1use std::sync::Arc;
63use std::time::Duration;
64
65use crate::control::{
66 ControlPlane, CreateImageRequest, Image, Microvm, RunHookPayload, RunMicrovmRequest, WaitOpts,
67};
68use crate::error::{Error, ErrorKind};
69use crate::region::Region;
70use crate::session::{Session, TokenMinter};
71
72pub const DEFAULT_RUNNING_TIMEOUT: Duration = Duration::from_secs(300);
76
77#[deprecated(
79 note = "renamed DEFAULT_RUNNING_TIMEOUT: `session::DEFAULT_READY_TIMEOUT` shared this name \
80 for a different wait, the daemon's"
81)]
82pub const DEFAULT_READY_TIMEOUT: Duration = DEFAULT_RUNNING_TIMEOUT;
83
84pub const DEFAULT_LIFECYCLE_TIMEOUT: Duration = Duration::from_secs(300);
87
88pub const DEFAULT_DELETE_ATTEMPTS: u32 = 20;
94
95pub const DEFAULT_DELETE_BACKOFF: Duration = Duration::from_secs(15);
97
98pub const DENY_EGRESS_PROXY_URL: &str = "http://127.0.0.1:1";
104
105pub const DENY_EGRESS_ENV_KEYS: [&str; 6] = [
114 "http_proxy",
115 "https_proxy",
116 "all_proxy",
117 "HTTP_PROXY",
118 "HTTPS_PROXY",
119 "ALL_PROXY",
120];
121
122pub const LIFECYCLE_POLL_INTERVAL: Duration = Duration::from_secs(5);
125
126#[derive(Clone, Copy, Debug, Eq, PartialEq)]
132pub enum Lifecycle {
133 Pending,
135 Running,
137 Suspending,
139 Suspended,
141 Terminating,
143 Terminated,
145}
146
147impl Lifecycle {
148 pub fn as_str(self) -> &'static str {
150 match self {
151 Lifecycle::Pending => "PENDING",
152 Lifecycle::Running => "RUNNING",
153 Lifecycle::Suspending => "SUSPENDING",
154 Lifecycle::Suspended => "SUSPENDED",
155 Lifecycle::Terminating => "TERMINATING",
156 Lifecycle::Terminated => "TERMINATED",
157 }
158 }
159
160 pub fn from_service(state: &str) -> Option<Self> {
163 Some(match state {
164 "PENDING" => Lifecycle::Pending,
165 "RUNNING" => Lifecycle::Running,
166 "SUSPENDING" => Lifecycle::Suspending,
167 "SUSPENDED" => Lifecycle::Suspended,
168 "TERMINATING" => Lifecycle::Terminating,
169 "TERMINATED" => Lifecycle::Terminated,
170 _ => return None,
171 })
172 }
173
174 pub fn is_live(self) -> bool {
176 matches!(
177 self,
178 Lifecycle::Pending | Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended
179 )
180 }
181}
182
183impl std::fmt::Display for Lifecycle {
184 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
185 f.write_str(self.as_str())
186 }
187}
188
189#[derive(Clone, Debug)]
195pub struct RunRequest {
196 pub image_identifier: Option<String>,
202 pub image_version: Option<String>,
212 pub execution_role_arn: Option<String>,
214 pub agent_token: Option<String>,
216 pub client_token: Option<String>,
220 pub launch_env: std::collections::HashMap<String, String>,
234 pub identity: bool,
246 pub egress: bool,
250 pub egress_network_connectors: Vec<String>,
253 pub deny_egress: bool,
260 pub shell: bool,
265 pub max_idle_sec: u32,
267 pub suspended_sec: u32,
269 pub auto_resume: bool,
271 pub max_duration_sec: u32,
273 pub ready_timeout: Duration,
276 pub token_scope: Option<String>,
278 pub logging: Option<crate::control::ops::Logging>,
280 pub wait: bool,
287}
288
289impl Default for RunRequest {
290 fn default() -> Self {
291 Self {
292 image_identifier: None,
293 image_version: None,
294 execution_role_arn: None,
295 agent_token: None,
296 client_token: None,
297 launch_env: std::collections::HashMap::new(),
298 identity: false,
299 egress: false,
300 egress_network_connectors: Vec::new(),
301 deny_egress: false,
302 shell: false,
303 max_idle_sec: 600,
304 suspended_sec: 600,
305 auto_resume: false,
306 max_duration_sec: 3_600,
307 ready_timeout: DEFAULT_RUNNING_TIMEOUT,
308 token_scope: None,
309 logging: None,
310 wait: true,
311 }
312 }
313}
314
315impl RunRequest {
316 pub fn new() -> Self {
319 Self::default()
320 }
321
322 #[must_use]
324 pub fn with_image(mut self, identifier: impl Into<String>) -> Self {
325 self.image_identifier = Some(identifier.into());
326 self
327 }
328
329 #[must_use]
331 pub fn with_egress(mut self) -> Self {
332 self.egress = true;
333 self
334 }
335
336 #[must_use]
338 pub fn with_egress_network_connector(mut self, arn: impl Into<String>) -> Self {
339 self.egress_network_connectors.push(arn.into());
340 self
341 }
342
343 #[must_use]
349 pub fn with_vpc_egress(mut self, arns: Vec<String>) -> Self {
350 if !arns.is_empty() {
351 self.egress = false;
352 self.egress_network_connectors = arns;
353 }
354 self
355 }
356
357 #[must_use]
360 pub fn with_deny_egress(mut self) -> Self {
361 self.deny_egress = true;
362 self
363 }
364
365 pub fn egress_posture(&self) -> crate::control::EgressPosture {
370 crate::control::EgressPosture::for_launch(self.egress, self.deny_egress)
371 }
372
373 pub fn effective_launch_env(&self) -> std::collections::HashMap<String, String> {
381 let mut env = self.launch_env.clone();
382 if self.deny_egress {
383 for key in DENY_EGRESS_ENV_KEYS {
384 env.entry(key.to_string())
385 .or_insert_with(|| DENY_EGRESS_PROXY_URL.to_string());
386 }
387 }
388 env
389 }
390
391 #[must_use]
393 pub fn with_shell(mut self) -> Self {
394 self.shell = true;
395 self
396 }
397
398 #[must_use]
400 pub fn with_image_version(mut self, version: impl Into<String>) -> Self {
401 self.image_version = Some(version.into());
402 self
403 }
404
405 #[must_use]
407 pub fn with_suspended_sec(mut self, seconds: u32) -> Self {
408 self.suspended_sec = seconds;
409 self
410 }
411
412 #[must_use]
414 pub fn with_agent_token(mut self, token: impl Into<String>) -> Self {
415 self.agent_token = Some(token.into());
416 self
417 }
418
419 #[must_use]
424 pub fn with_identity(mut self) -> Self {
425 self.identity = true;
426 self
427 }
428
429 #[must_use]
436 pub fn with_launch_env(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
437 self.launch_env.insert(key.into(), value.into());
438 self
439 }
440}
441
442#[derive(Clone, Copy, Debug)]
448pub struct TeardownOpts {
449 pub delete_image: bool,
451 pub delete_log_group: bool,
457 pub delete_attempts: u32,
459 pub delete_backoff: Duration,
461 pub wait_for_terminated: Option<Duration>,
469}
470
471impl Default for TeardownOpts {
472 fn default() -> Self {
473 Self {
474 delete_image: false,
475 delete_log_group: false,
476 delete_attempts: DEFAULT_DELETE_ATTEMPTS,
477 delete_backoff: DEFAULT_DELETE_BACKOFF,
478 wait_for_terminated: None,
479 }
480 }
481}
482
483impl TeardownOpts {
484 #[must_use]
486 pub fn deleting_image(mut self) -> Self {
487 self.delete_image = true;
488 self
489 }
490
491 #[must_use]
493 pub fn deleting_log_group(mut self) -> Self {
494 self.delete_log_group = true;
495 self
496 }
497
498 #[must_use]
500 pub fn waiting_for_terminated(mut self) -> Self {
501 self.wait_for_terminated = Some(DEFAULT_LIFECYCLE_TIMEOUT);
502 self
503 }
504}
505
506#[derive(Clone, Debug, Default, Eq, PartialEq)]
510pub struct TeardownReport {
511 pub undeleted: Vec<String>,
526 pub terminate_accepted: bool,
528 pub image_deleted: Option<bool>,
530 pub lifecycle: Option<Lifecycle>,
532 pub failures: Vec<String>,
537}
538
539impl TeardownReport {
540 pub fn leaked(&self) -> bool {
542 !self.undeleted.is_empty()
543 }
544}
545
546pub struct ControlPlaneMinter {
560 control: Arc<ControlPlane>,
561 microvm_id: String,
562}
563
564impl ControlPlaneMinter {
565 pub fn new(control: Arc<ControlPlane>, microvm_id: impl Into<String>) -> Self {
567 Self {
568 control,
569 microvm_id: microvm_id.into(),
570 }
571 }
572}
573
574impl std::fmt::Debug for ControlPlaneMinter {
575 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
576 f.debug_struct("ControlPlaneMinter")
577 .field("microvm_id", &self.microvm_id)
578 .finish_non_exhaustive()
579 }
580}
581
582impl TokenMinter for ControlPlaneMinter {
583 fn mint(
584 &self,
585 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
586 Box::pin(async move {
587 let minted = self.control.mint_auth_token(&self.microvm_id).await?;
588 Ok(minted.into())
589 })
590 }
591
592 fn mint_for_ports(
600 &self,
601 ports: &[u16],
602 ) -> futures_util::future::BoxFuture<'_, Result<crate::session::ProxyToken, Error>> {
603 let specs: Vec<crate::control::ops::PortSpecification> = ports
604 .iter()
605 .map(|port| crate::control::ops::PortSpecification::port(*port))
606 .collect();
607 Box::pin(async move {
608 let minted = self
609 .control
610 .mint_auth_token_for(&self.microvm_id, &specs)
611 .await?;
612 Ok(minted.into())
613 })
614 }
615}
616
617pub struct Sandbox {
621 control: Arc<ControlPlane>,
622 image: Option<Image>,
623 launch_image_arn: Option<String>,
625 microvm: Option<Microvm>,
626 session: Option<Session>,
627 session_backend: Option<crate::session::SharedBackend>,
634
635 lifecycle: Lifecycle,
637 lifecycle_watch: tokio::sync::watch::Sender<Lifecycle>,
640 token_installed: bool,
641 image_exists: bool,
642 was_terminated: bool,
643 bootstrap_count: u32,
644
645 suspended_window: Option<Duration>,
648 launch_adoptable: bool,
651 launch_posture: Option<crate::control::EgressPosture>,
654 pending_agent_token: Option<String>,
657 idle_window: Option<Duration>,
660 suspended_at: Option<Duration>,
662 torn_down: bool,
665 adopted: bool,
668 detached: bool,
671 tunnel_identity: Option<crate::identity::TunnelIdentity>,
677 build_services: Option<Arc<dyn crate::control::BuildServices>>,
680 account: Option<String>,
682}
683
684impl std::fmt::Debug for Sandbox {
685 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
686 f.debug_struct("Sandbox")
689 .field("lifecycle", &self.lifecycle)
690 .field("microvm", &self.microvm.as_ref().map(|vm| &vm.id))
691 .field("image", &self.image.as_ref().map(|image| &image.identifier))
692 .field("token_installed", &self.token_installed)
693 .field("bootstrap_count", &self.bootstrap_count)
694 .field("was_terminated", &self.was_terminated)
695 .field("suspended_window", &self.suspended_window)
696 .field("adopted", &self.adopted)
697 .field("detached", &self.detached)
698 .finish_non_exhaustive()
699 }
700}
701
702#[derive(Clone)]
709pub struct Detached {
710 pub microvm_id: String,
712 pub endpoint: String,
714 pub region: Region,
716 pub port: u16,
718 agent_token: String,
719}
720
721impl Detached {
722 pub fn agent_token(&self) -> &str {
724 &self.agent_token
725 }
726}
727
728impl std::fmt::Debug for Detached {
729 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
730 f.debug_struct("Detached")
731 .field("microvm_id", &self.microvm_id)
732 .field("endpoint", &self.endpoint)
733 .field("region", &self.region)
734 .field("port", &self.port)
735 .field("agent_token", &"<redacted>")
736 .finish()
737 }
738}
739
740impl Sandbox {
741 pub fn with_control_plane(control: ControlPlane) -> Self {
755 Self {
756 control: Arc::new(control),
757 image: None,
758 launch_image_arn: None,
759 microvm: None,
760 session: None,
761 session_backend: None,
762 lifecycle: Lifecycle::Pending,
763 lifecycle_watch: tokio::sync::watch::Sender::new(Lifecycle::Pending),
764 token_installed: false,
765 image_exists: false,
766 was_terminated: false,
767 bootstrap_count: 0,
768 suspended_window: None,
769 launch_adoptable: false,
770 launch_posture: None,
771 pending_agent_token: None,
772 idle_window: None,
773 suspended_at: None,
774 torn_down: false,
775 adopted: false,
776 detached: false,
777 tunnel_identity: None,
778 build_services: None,
779 account: None,
780 }
781 }
782
783 pub fn with_build_services(mut self, services: Arc<dyn crate::control::BuildServices>) -> Self {
789 self.build_services = Some(services);
790 self
791 }
792
793 pub fn with_session_backend(mut self, backend: crate::session::SharedBackend) -> Self {
799 self.session_backend = Some(backend);
800 self
801 }
802
803 pub fn lifecycle(&self) -> Lifecycle {
807 self.lifecycle
808 }
809
810 pub fn token_installed(&self) -> bool {
812 self.token_installed
813 }
814
815 pub fn image_exists(&self) -> bool {
817 self.image_exists
818 }
819
820 pub fn was_terminated(&self) -> bool {
822 self.was_terminated
823 }
824
825 pub fn bootstrap_count(&self) -> u32 {
827 self.bootstrap_count
828 }
829
830 pub fn image(&self) -> Option<&Image> {
832 self.image.as_ref()
833 }
834
835 pub fn launch_image_arn(&self) -> Option<&str> {
844 self.launch_image_arn.as_deref()
845 }
846
847 pub fn microvm(&self) -> Option<&Microvm> {
849 self.microvm.as_ref()
850 }
851
852 pub fn session(&self) -> Option<&Session> {
854 self.session.as_ref()
855 }
856
857 pub fn suspended_window(&self) -> Option<Duration> {
859 self.suspended_window
860 }
861
862 pub fn watch_lifecycle(&self) -> tokio::sync::watch::Receiver<Lifecycle> {
869 self.lifecycle_watch.subscribe()
870 }
871
872 fn set_lifecycle(&mut self, lifecycle: Lifecycle) {
873 self.lifecycle = lifecycle;
874 self.lifecycle_watch.send_replace(lifecycle);
875 }
876
877 pub fn idle_window(&self) -> Option<Duration> {
879 self.idle_window
880 }
881
882 pub fn tunnel_identity(&self) -> Option<&crate::identity::TunnelIdentity> {
888 self.tunnel_identity.as_ref()
889 }
890
891 pub fn egress_posture(&self) -> crate::control::EgressPosture {
899 self.launch_posture.unwrap_or_default()
900 }
901
902 pub fn port(&self) -> u16 {
907 self.control.port()
908 }
909
910 pub async fn build_image(&mut self, request: CreateImageRequest) -> Result<&Image, Error> {
918 let size = request.size;
919 let created = self.control.create_image(request).await?;
920 let mut built = self
921 .control
922 .wait_for_image(&created.identifier, size, WaitOpts::default())
923 .await?;
924 built.log_group = created.log_group;
930 built.log_stream = created.log_stream;
931 self.image_exists = true;
934 self.image = Some(built);
935 Ok(self.image.as_ref().expect("just assigned"))
936 }
937
938 pub async fn ensure_image(
942 &mut self,
943 request: crate::control::EnsureImageRequest,
944 ) -> Result<crate::control::EnsuredImage, Error> {
945 let control = Arc::clone(&self.control);
954 let prepared =
955 tokio::task::spawn_blocking(move || crate::control::ensure::prepare(&control, request))
956 .await
957 .map_err(|joined| {
958 Error::new(
959 ErrorKind::Unexpected,
960 format!("preparing the image's artifact did not complete: {joined}"),
961 )
962 })??;
963 let services = self.build_services().await?;
964 let account = match &self.account {
965 Some(account) => account.clone(),
966 None => {
967 let account = services.caller_account().await?;
968 self.account = Some(account.clone());
969 account
970 }
971 };
972 let ensured =
973 crate::control::ensure::ensure(&self.control, services.as_ref(), &account, prepared)
974 .await?;
975 self.image_exists = true;
976 self.image = Some(ensured.image.clone());
977 Ok(ensured)
978 }
979
980 async fn build_services(&mut self) -> Result<Arc<dyn crate::control::BuildServices>, Error> {
983 if let Some(services) = &self.build_services {
984 return Ok(Arc::clone(services));
985 }
986 let built = self
987 .control
988 .adapters()
989 .build_services(self.control.region().clone())
990 .await?;
991 self.build_services = Some(Arc::clone(&built));
992 Ok(built)
993 }
994
995 pub fn build_artifact_for(&self, request: &CreateImageRequest) -> Result<Vec<u8>, Error> {
1000 self.control.build_artifact_for(request)
1001 }
1002
1003 pub async fn upload_artifact_for(&mut self, request: &CreateImageRequest) -> Result<(), Error> {
1009 let (bucket, key) = crate::control::ensure::s3_location(&request.code_artifact_uri)?;
1010 let bytes = self.control.build_artifact_for(request)?;
1011 let services = self.build_services().await?;
1012 services.put_object(bucket, key, bytes).await
1013 }
1014
1015 pub fn preflight(&self, request: &CreateImageRequest) -> Result<(), Error> {
1022 self.control.preflight(request)
1023 }
1024
1025 pub async fn resolve_image_arn(&self, identifier: &str) -> Result<String, Error> {
1033 self.control.resolve_image_arn(identifier).await
1034 }
1035
1036 pub async fn find_image_by_name(
1039 &self,
1040 name: &str,
1041 ) -> Result<Option<crate::control::ops::MicrovmImageSummaryWire>, Error> {
1042 self.control.find_image_by_name(name).await
1043 }
1044
1045 pub fn artifact_content_hash_for(&self, request: &CreateImageRequest) -> String {
1047 self.control.artifact_content_hash_for(request)
1048 }
1049
1050 pub fn dockerfile_for(&self, request: &CreateImageRequest) -> String {
1053 self.control.dockerfile_for(request)
1054 }
1055
1056 pub async fn managed_base_versions(
1064 &self,
1065 base_image_arn: &str,
1066 ) -> Result<Vec<crate::control::ops::ManagedMicrovmImageVersionWire>, Error> {
1067 self.control.managed_base_versions(base_image_arn).await
1068 }
1069
1070 pub async fn image_version(
1072 &self,
1073 identifier: &str,
1074 version: &str,
1075 ) -> Result<crate::control::ops::MicrovmImageVersionSummaryWire, Error> {
1076 self.control.get_image_version(identifier, version).await
1077 }
1078
1079 pub async fn run(&mut self, request: RunRequest) -> Result<&mut Session, Error> {
1106 self.refuse_detached("run")?;
1107 if self.bootstrap_count > 0 || self.microvm.is_some() {
1111 return Err(Error::invalid_arg(format!(
1112 "this sandbox has already launched a VM ({} bootstrap(s), lifecycle {}), and the \
1113 agent token is installed at most once per VM lifetime (STATE-3). A second VM \
1114 needs a second Sandbox — reusing this one would either re-deliver a run-hook \
1115 payload to a daemon whose one-shot bootstrap refuses it, or silently address \
1116 two guests through one handle.",
1117 self.bootstrap_count, self.lifecycle,
1118 )));
1119 }
1120
1121 let posture = crate::control::egress_posture_for(
1128 request.egress,
1129 &request.egress_network_connectors,
1130 request.deny_egress,
1131 Some(self.control.region()),
1132 )?;
1133
1134 let Some(identifier) = request
1135 .image_identifier
1136 .clone()
1137 .or_else(|| self.image.as_ref().map(|image| image.identifier.clone()))
1138 else {
1139 return Err(Error::new(
1140 ErrorKind::Precondition,
1141 "no image to launch: pass RunRequest::with_image or call build_image first."
1142 .to_string(),
1143 ));
1144 };
1145
1146 if request.client_token.is_some() && (request.agent_token.is_none() || request.identity) {
1147 return Err(Error::invalid_arg(
1148 "a stable client_token requires an explicit agent_token and identity=false; persist identical launch parameters before retrying",
1149 ));
1150 }
1151 let agent_token = match request.agent_token.clone() {
1152 Some(token) => token,
1153 None => mint_agent_token(self.control.entropy())?,
1154 };
1155 let launch_identity = if request.identity {
1159 Some(crate::entropy::launch_identity(self.control.entropy())?)
1160 } else {
1161 None
1162 };
1163 let payload = RunHookPayload::for_launch_with_identity(
1170 &agent_token,
1171 &request.effective_launch_env(),
1172 launch_identity.as_ref(),
1173 )?;
1174
1175 let mut wire = RunMicrovmRequest::new(identifier, payload);
1176 wire.image_version = request.image_version.clone();
1177 wire.execution_role_arn = request.execution_role_arn.clone();
1178 wire.egress_network_connectors = request.egress_network_connectors.clone();
1179 wire.max_idle_sec = request.max_idle_sec;
1180 wire.suspended_sec = request.suspended_sec;
1181 wire.auto_resume = request.auto_resume;
1182 wire.max_duration_sec = request.max_duration_sec;
1183 wire.token_scope = request.token_scope.clone();
1184 wire.client_token = request.client_token.clone();
1185 wire.logging = request.logging.clone();
1186 if request.egress {
1187 wire = wire.with_egress();
1188 }
1189 if request.shell {
1190 wire = wire.with_shell();
1191 }
1192
1193 self.control.launch_connectors(&wire)?;
1199 wire.image_identifier = self
1200 .control
1201 .resolve_image_arn(&wire.image_identifier)
1202 .await?;
1203 self.launch_image_arn = Some(wire.image_identifier.clone());
1204
1205 let launched = self.control.run_microvm(wire).await?;
1206
1207 self.tunnel_identity = launch_identity.map(crate::identity::LaunchIdentity::keep);
1211
1212 self.set_lifecycle(Lifecycle::Pending);
1214 self.image_exists = true;
1215 self.suspended_window = Some(Duration::from_secs(u64::from(request.suspended_sec)));
1219 self.idle_window = Some(Duration::from_secs(u64::from(request.max_idle_sec)));
1220 self.microvm = Some(launched);
1221 self.launch_adoptable = request.client_token.is_some();
1224 self.launch_posture = Some(posture);
1225 self.pending_agent_token = Some(agent_token);
1226
1227 if request.wait {
1228 return self.wait_until_running(request.ready_timeout).await;
1229 }
1230 self.build_session()?;
1233 Ok(self.session.as_mut().expect("just assigned"))
1234 }
1235
1236 fn build_session(&mut self) -> Result<(), Error> {
1238 let (Some(vm), Some(agent_token)) = (&self.microvm, &self.pending_agent_token) else {
1239 return Err(Error::new(
1240 ErrorKind::Precondition,
1241 "no launch to build a session for",
1242 ));
1243 };
1244 let minter = Arc::new(ControlPlaneMinter::new(
1245 Arc::clone(&self.control),
1246 vm.id.clone(),
1247 ));
1248 let mut builder = Session::builder(vm.endpoint.clone(), agent_token.clone())
1251 .with_minter(minter)
1252 .with_port(self.control.port())
1253 .with_egress_posture(self.egress_posture())
1254 .with_clock(self.control.shared_clock());
1255 if let Some(backend) = &self.session_backend {
1256 builder = builder.with_backend(Arc::clone(backend));
1257 }
1258 self.session = Some(builder.build_with(self.control.adapters())?);
1259 Ok(())
1260 }
1261
1262 pub async fn wait_until_running(&mut self, timeout: Duration) -> Result<&mut Session, Error> {
1278 self.refuse_detached("wait until running")?;
1279 let id = self.require_microvm("wait_until_running")?;
1280 if self.lifecycle != Lifecycle::Pending {
1281 return Err(Error::invalid_arg(format!(
1282 "microvm {id} is {}; wait_until_running finishes a launch that is still PENDING",
1283 self.lifecycle,
1284 )));
1285 }
1286 let opts = WaitOpts::for_lifecycle(timeout);
1287 let running = if self.launch_adoptable {
1288 self.control.wait_for_launch(&id, opts).await?
1289 } else {
1290 self.control.wait_for_running(&id, opts).await?
1291 };
1292
1293 self.set_lifecycle(Lifecycle::Running);
1296 self.token_installed = true;
1297 self.bootstrap_count += 1;
1298 self.microvm = Some(running);
1299 if self.session.is_none() {
1300 self.build_session()?;
1301 }
1302 let session = self.session.as_mut().expect("just assigned");
1303 session
1304 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1305 .await?;
1306 Ok(session)
1307 }
1308
1309 pub async fn wait_until_ready(
1319 &mut self,
1320 running_timeout: Duration,
1321 ) -> Result<&mut Session, Error> {
1322 if self.lifecycle == Lifecycle::Running {
1323 self.refuse_detached("wait until ready")?;
1324 let session = self.session.as_mut().ok_or_else(|| {
1325 Error::new(ErrorKind::Precondition, "a RUNNING sandbox with no session")
1326 })?;
1327 session
1328 .wait_until_ready(crate::session::DEFAULT_BOOTSTRAP_TIMEOUT)
1329 .await?;
1330 return Ok(session);
1331 }
1332 self.wait_until_running(running_timeout).await
1333 }
1334
1335 pub async fn adopt(
1361 control: ControlPlane,
1362 microvm_id: impl Into<String>,
1363 endpoint: impl Into<String>,
1364 agent_token: impl Into<String>,
1365 ) -> Result<Self, Error> {
1366 let microvm_id = microvm_id.into();
1367 let endpoint = endpoint.into();
1368 let agent_token = agent_token.into();
1369 if agent_token.is_empty() {
1370 return Err(Error::invalid_arg(
1371 "adopt needs the agent token the VM was launched with: its daemon accepts only \
1372 that bearer, and adopting without it would build a handle every exec refuses.",
1373 ));
1374 }
1375 let mut vm = control.get_microvm(µvm_id).await?;
1376 if !endpoint.is_empty() && !vm.endpoint.is_empty() && vm.endpoint != endpoint {
1377 return Err(Error::invalid_arg(format!(
1378 "microvm {microvm_id} reports endpoint {}, not {endpoint}: the id and endpoint \
1379 came from different records, and a session built from them would address one \
1380 VM while its lifecycle calls reached another.",
1381 vm.endpoint,
1382 )));
1383 }
1384 if vm.endpoint.is_empty() {
1385 vm.endpoint = endpoint;
1386 }
1387 let Some(lifecycle) = Lifecycle::from_service(&vm.state) else {
1388 return Err(Error::new(
1389 ErrorKind::Platform,
1390 format!(
1391 "microvm {microvm_id} reports state {}, which this client does not know; \
1392 refusing to guess which lifecycle guards apply.",
1393 vm.state,
1394 ),
1395 ));
1396 };
1397
1398 let mut sandbox = Self::with_control_plane(control);
1399 sandbox.adopted = true;
1400 sandbox.set_lifecycle(lifecycle);
1401 sandbox.idle_window = vm
1404 .idle_policy
1405 .as_ref()
1406 .map(|policy| Duration::from_secs(u64::from(policy.max_idle_duration_seconds)));
1407 sandbox.image_exists = true;
1409 sandbox.launch_adoptable = true;
1411 sandbox.pending_agent_token = Some(agent_token);
1412 match lifecycle {
1413 Lifecycle::Pending => {}
1414 Lifecycle::Running | Lifecycle::Suspending | Lifecycle::Suspended => {
1415 sandbox.token_installed = true;
1416 sandbox.bootstrap_count = 1;
1417 }
1418 Lifecycle::Terminating | Lifecycle::Terminated => {
1419 sandbox.bootstrap_count = 1;
1420 sandbox.was_terminated = true;
1421 }
1422 }
1423 sandbox.microvm = Some(vm);
1424 if lifecycle.is_live() {
1425 sandbox.build_session()?;
1426 }
1427 Ok(sandbox)
1428 }
1429
1430 pub fn name_record(&self, name: &str) -> Result<crate::names::NameRecord, Error> {
1437 let Some(vm) = self.microvm() else {
1438 return Err(Error::new(
1439 ErrorKind::Precondition,
1440 "there is no VM to name yet: launch or adopt one first",
1441 ));
1442 };
1443 let Some(session) = self.session() else {
1444 return Err(Error::new(
1445 ErrorKind::Precondition,
1446 format!(
1447 "microvm {} has no session to take its agent token from",
1448 vm.id
1449 ),
1450 ));
1451 };
1452 let mut record = crate::names::NameRecord::new_at(
1454 name,
1455 vm.id.as_str(),
1456 vm.endpoint.as_str(),
1457 session.agent_token(),
1458 self.control.region().as_str(),
1459 self.control.clock().unix_now().as_secs(),
1460 )?;
1461 record.set_tunnel_identity(self.tunnel_identity.as_ref());
1462 Ok(record)
1463 }
1464
1465 pub async fn adopt_record(
1472 control: ControlPlane,
1473 record: crate::names::NameRecord,
1474 ) -> Result<Self, Error> {
1475 let identity = record.tunnel_identity()?;
1476 let mut sandbox = Self::adopt(
1477 control,
1478 record.microvm_id,
1479 record.endpoint,
1480 record.agent_token,
1481 )
1482 .await?;
1483 sandbox.tunnel_identity = identity;
1484 Ok(sandbox)
1485 }
1486
1487 pub fn detach(&mut self) -> Result<Detached, Error> {
1500 self.refuse_detached("detach")?;
1501 let Some(vm) = self.microvm.as_ref() else {
1502 return Err(Error::new(
1503 ErrorKind::Precondition,
1504 "nothing to detach: this sandbox has not launched a VM.",
1505 ));
1506 };
1507 if self.torn_down || !self.lifecycle.is_live() {
1508 return Err(Error::new(
1509 ErrorKind::Precondition,
1510 format!(
1511 "microvm {} is {} and there is no live VM to hand off; only a PENDING, \
1512 RUNNING, or SUSPENDED VM can be adopted.",
1513 vm.id, self.lifecycle,
1514 ),
1515 ));
1516 }
1517 let agent_token = self
1518 .session
1519 .as_ref()
1520 .map(|session| session.agent_token().to_string())
1521 .or_else(|| self.pending_agent_token.clone())
1522 .ok_or_else(|| {
1523 Error::new(
1524 ErrorKind::Precondition,
1525 format!(
1526 "microvm {} has no agent token to hand off; an adopter could not \
1527 reach its daemon.",
1528 vm.id
1529 ),
1530 )
1531 })?;
1532 let detached = Detached {
1533 microvm_id: vm.id.clone(),
1534 endpoint: vm.endpoint.clone(),
1535 region: self.control.region().clone(),
1536 port: self.control.port(),
1537 agent_token,
1538 };
1539 self.detached = true;
1540 self.session = None;
1541 self.pending_agent_token = None;
1542 Ok(detached)
1543 }
1544
1545 pub fn detached(&self) -> bool {
1547 self.detached
1548 }
1549
1550 fn refuse_detached(&self, what: &str) -> Result<(), Error> {
1552 if !self.detached {
1553 return Ok(());
1554 }
1555 let id = self.microvm.as_ref().map_or("?", |vm| vm.id.as_str());
1556 Err(Error::new(
1557 ErrorKind::Precondition,
1558 format!(
1559 "cannot {what}: microvm {id} was detached and belongs to whoever adopts it. \
1560 Adopt it with `Sandbox::adopt` and {what} through that handle."
1561 ),
1562 ))
1563 }
1564
1565 pub fn adopted(&self) -> bool {
1567 self.adopted
1568 }
1569
1570 pub async fn suspend(&mut self) -> Result<(), Error> {
1585 self.refuse_detached("suspend")?;
1586 let id = self.require_microvm("suspend")?;
1587
1588 if self.lifecycle != Lifecycle::Running {
1590 return Err(Error::invalid_arg(format!(
1591 "microvm {id} is {} and a suspend is only issued from RUNNING (STATE-5). Refused \
1592 here rather than by the service, because the service's answer about a \
1593 non-running id does not say which of the two things went wrong — and a suspend \
1594 issued from SUSPENDED is a caller who believes they resumed.",
1595 self.lifecycle,
1596 )));
1597 }
1598
1599 self.control.suspend(&id).await?;
1604 self.set_lifecycle(Lifecycle::Suspending);
1605 self.suspended_at = Some(self.control.clock().elapsed());
1609
1610 let settled = self
1611 .control
1612 .wait_for_state(
1613 &id,
1614 &crate::control::microvm::SUSPEND_WANTED,
1615 &[],
1616 self.lifecycle_wait(),
1617 )
1618 .await?;
1619
1620 let settled_lifecycle = match settled.state.as_str() {
1625 "SUSPENDED" => Lifecycle::Suspended,
1626 "TERMINATED" => {
1627 self.was_terminated = true;
1628 Lifecycle::Terminated
1629 }
1630 other => {
1631 return Err(Error::new(
1632 ErrorKind::Platform,
1633 format!(
1634 "the suspend wait returned {other}, which is neither SUSPENDED nor \
1635 TERMINATED — and those two are what this client asked for."
1636 ),
1637 ));
1638 }
1639 };
1640 self.set_lifecycle(settled_lifecycle);
1641 self.microvm = Some(settled);
1642 Ok(())
1643 }
1644
1645 pub async fn resume(&mut self) -> Result<&mut Session, Error> {
1669 self.refuse_detached("resume")?;
1670 let id = self.require_microvm("resume")?;
1671
1672 if self.was_terminated || self.lifecycle == Lifecycle::Terminated {
1675 return Err(Error::invalid_arg(format!(
1676 "microvm {id} was terminated, and a terminated VM never returns to RUNNING \
1677 (STATE-11). There is nothing to resume: the guest's memory is gone, so even a \
1678 call the service accepted would hand back a different machine."
1679 )));
1680 }
1681 if self.lifecycle != Lifecycle::Suspended {
1682 return Err(Error::invalid_arg(format!(
1683 "microvm {id} is {} and a resume is only issued from SUSPENDED (STATE-7).",
1684 self.lifecycle,
1685 )));
1686 }
1687
1688 self.require_open_suspended_window(&id)?;
1690
1691 self.control.resume(&id).await?;
1692 let running = self
1698 .control
1699 .wait_for_state(
1700 &id,
1701 &["RUNNING"],
1702 &crate::constants::DEAD_STATES,
1703 self.lifecycle_wait(),
1704 )
1705 .await?;
1706
1707 self.set_lifecycle(Lifecycle::Running);
1708 let endpoint = running.endpoint.clone();
1712 self.microvm = Some(running);
1713 if let Some(session) = self.session.as_mut() {
1714 session.rebind(endpoint);
1715 }
1716 self.suspended_at = None;
1720
1721 self.session.as_mut().ok_or_else(|| {
1722 Error::new(
1723 ErrorKind::Unexpected,
1724 "the VM resumed but this sandbox holds no session, which run() always builds"
1725 .to_string(),
1726 )
1727 })
1728 }
1729
1730 fn require_open_suspended_window(&self, id: &str) -> Result<(), Error> {
1735 let reported = self
1736 .microvm
1737 .as_ref()
1738 .and_then(|vm| vm.idle_policy.as_ref())
1739 .map(|policy| Duration::from_secs(u64::from(policy.suspended_duration_seconds)));
1740 let (Some(window), Some(since)) = (self.suspended_window.or(reported), self.suspended_at)
1741 else {
1742 return Ok(());
1745 };
1746 let elapsed = self.control.clock().elapsed().saturating_sub(since);
1747 if elapsed <= window {
1748 return Ok(());
1749 }
1750 Err(Error::new(
1751 ErrorKind::WindowClosed,
1752 format!(
1753 "microvm {id} has been suspended {}s, past the {}s suspendedDurationSeconds \
1754 window set at launch — the idlePolicy terminates a suspended VM once that window \
1755 passes, so there is nothing left to resume (docs/PLATFORM.md, '`idlePolicy`'). \
1756 Refused before ResumeMicrovm, because calling would cost the full poll timeout \
1757 to learn the same thing less clearly. A longer window has to be set at launch \
1758 on the next VM; there is no call that extends this one.",
1759 elapsed.as_secs(),
1760 window.as_secs(),
1761 ),
1762 ))
1763 }
1764
1765 pub async fn terminate(&mut self, opts: TeardownOpts) -> TeardownReport {
1773 let mut report = TeardownReport::default();
1774 if let Err(error) = self.refuse_detached("terminate") {
1775 report.failures.push(error.to_string());
1778 report.lifecycle = Some(self.lifecycle);
1779 return report;
1780 }
1781 self.torn_down = true;
1782
1783 self.session = None;
1786
1787 if let Some(id) = self.microvm.as_ref().map(|vm| vm.id.clone()) {
1789 self.set_lifecycle(Lifecycle::Terminating);
1793 self.was_terminated = true;
1794
1795 match self.control.terminate(&id).await {
1796 Ok(()) => report.terminate_accepted = true,
1797 Err(error) => {
1798 report.failures.push(format!("terminate {id}: {error}"));
1799 report.undeleted.push(id.clone());
1800 }
1801 }
1802
1803 if report.terminate_accepted
1804 && let Some(timeout) = opts.wait_for_terminated
1805 {
1806 let wait = WaitOpts::for_lifecycle(timeout);
1807 match self
1808 .control
1809 .wait_for_state(&id, &["TERMINATED"], &[], wait)
1810 .await
1811 {
1812 Ok(settled) => {
1814 self.set_lifecycle(Lifecycle::Terminated);
1815 self.microvm = Some(settled);
1816 }
1817 Err(error) => report
1820 .failures
1821 .push(format!("waiting for {id} to reach TERMINATED: {error}")),
1822 }
1823 }
1824 }
1825
1826 if opts.delete_image {
1829 match self.image.as_ref().map(|image| image.identifier.clone()) {
1830 Some(identifier) => {
1831 let deleted = self
1832 .control
1833 .delete_image(&identifier, opts.delete_attempts, opts.delete_backoff)
1834 .await;
1835 report.image_deleted = Some(deleted);
1836 if deleted {
1837 self.image_exists = false;
1838 } else {
1839 report.failures.push(format!(
1840 "the image {identifier} survived {} delete attempts",
1841 opts.delete_attempts.max(1)
1842 ));
1843 report.undeleted.push(identifier);
1844 }
1845 }
1846 None => report.image_deleted = Some(false),
1847 }
1848 }
1849
1850 if opts.delete_log_group
1852 && let Some(group) = self.image.as_ref().map(Image::build_log_group)
1853 {
1854 report.failures.push(format!(
1855 "the build log group {group} was not deleted: CloudWatch Logs is not in this \
1856 crate's dependency set, so it is reported rather than removed. It is \
1857 service-created, which means no Terraform stack owns it and `terraform destroy` \
1858 leaves it behind — six accumulated before anyone noticed."
1859 ));
1860 report.undeleted.push(group);
1861 }
1862
1863 report.lifecycle = Some(self.lifecycle);
1864 report
1865 }
1866
1867 fn require_microvm(&self, what: &str) -> Result<String, Error> {
1871 self.microvm
1872 .as_ref()
1873 .map(|vm| vm.id.clone())
1874 .ok_or_else(|| {
1875 Error::new(
1876 ErrorKind::Precondition,
1877 format!("nothing to {what}: this sandbox has not launched a VM."),
1878 )
1879 })
1880 }
1881
1882 fn lifecycle_wait(&self) -> WaitOpts {
1884 WaitOpts::for_lifecycle(DEFAULT_LIFECYCLE_TIMEOUT)
1885 }
1886}
1887
1888impl Sandbox {
1897 fn drop_warning(&self) -> Option<String> {
1901 if self.torn_down || self.adopted || self.detached {
1902 return None;
1903 }
1904 let vm = self.microvm.as_ref()?;
1905 self.lifecycle.is_live().then(|| {
1906 format!(
1907 "warning: the Sandbox for microvm {} was dropped in {} without terminate() or \
1908 detach(). Nothing was torn down — Drop cannot await, so a teardown here would \
1909 deadlock inside a runtime. The VM bills until its maximumDurationInSeconds \
1910 ceiling: terminate it with `microvm terminate {}`.",
1911 vm.id, self.lifecycle, vm.id,
1912 )
1913 })
1914 }
1915}
1916
1917impl Drop for Sandbox {
1918 fn drop(&mut self) {
1919 if let Some(warning) = self.drop_warning() {
1920 self.control.adapters().warn(&warning);
1921 }
1922 }
1923}
1924
1925fn mint_agent_token(entropy: &dyn crate::entropy::Entropy) -> Result<String, Error> {
1931 let mut bytes = [0u8; 32];
1932 entropy.fill(&mut bytes)?;
1933 Ok(const_hex::encode(bytes))
1934}
1935
1936#[cfg(test)]
1937mod tests {
1938 use super::*;
1939 use crate::control::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1940 use crate::control::transport::Method;
1941
1942 fn planted() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
1948 let recorder = Arc::new(FakeControlPlane::new());
1949 let clock = Arc::new(TestClock::new());
1950 let plane = crate::testing::control_plane(
1951 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
1952 Region::UsEast1,
1953 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
1954 );
1955 let sandbox = Sandbox::with_control_plane(plane)
1957 .with_session_backend(crate::testing::HealthyDaemon::new());
1958 (sandbox, recorder, clock)
1959 }
1960
1961 #[test]
1964 fn the_suspend_resume_and_terminate_waits_are_cores_lifecycle_wait() {
1965 let (sandbox, _, _) = planted();
1966 let wait = sandbox.lifecycle_wait();
1967 assert_eq!(
1968 (wait.timeout, wait.poll_interval, wait.stall_grace),
1969 (
1970 DEFAULT_LIFECYCLE_TIMEOUT,
1971 LIFECYCLE_POLL_INTERVAL,
1972 Duration::MAX
1973 )
1974 );
1975 }
1976
1977 fn answer_launch(recorder: &FakeControlPlane) {
1979 recorder
1980 .answer(
1981 "RunMicrovm",
1982 Answer::ok(fake::microvm_response("PENDING", None)),
1983 )
1984 .answer(
1985 "GetMicrovm",
1986 Answer::ok(fake::microvm_response("RUNNING", None)),
1987 )
1988 .answer(
1989 "CreateMicrovmAuthToken",
1990 Answer::ok(fake::auth_token_response("proxy-token")),
1991 );
1992 }
1993
1994 #[tokio::test]
1995 async fn vpc_egress_configuration_survives_the_sandbox_launch() {
1996 let (mut sandbox, recorder, _) = planted();
1997 answer_launch(&recorder);
1998 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
1999 let request = RunRequest::new()
2000 .with_image("arn:image")
2001 .with_egress_network_connector(arn);
2002 assert_eq!(
2003 request.egress_posture(),
2004 crate::control::EgressPosture::Unsealed
2005 );
2006 sandbox.run(request).await.expect("launch");
2007 let body = recorder.first_body("RunMicrovm");
2008 assert_eq!(body["egressNetworkConnectors"], serde_json::json!([arn]));
2009 }
2010
2011 #[tokio::test]
2017 async fn a_sandbox_session_reads_the_planes_clock() {
2018 let (mut sandbox, recorder, clock) = planted();
2019 answer_launch(&recorder);
2020 let session = sandbox
2021 .run(RunRequest::new().with_image("arn:image"))
2022 .await
2023 .expect("the launch reaches RUNNING");
2024 let pinned = Duration::from_secs(1_767_225_600) + Duration::from_nanos(987_654_321);
2025 clock.set_unix_now(pinned);
2026 let id = session.mint_exec_id();
2027 let value =
2028 u64::from_str_radix(id.strip_prefix("x-").expect("the x- shape"), 16).expect("hex");
2029 let low_bits = (1_u64 << 40) - 1;
2030 assert_eq!(
2031 value & low_bits,
2032 (pinned.as_nanos() as u64) & low_bits,
2033 "{id} wasn't minted from the plane's clock"
2034 );
2035 }
2036
2037 #[tokio::test]
2044 async fn ensure_image_builds_its_services_through_the_planes_adapters() {
2045 let (mut sandbox, recorder, _) = planted();
2046 let dockerfile = crate::control::artifact::wrap_dockerfile(
2047 "FROM python:3.12-slim\nWORKDIR /app\n",
2048 &crate::control::artifact::WrapOptions::default(),
2049 )
2050 .expect("a wrappable Dockerfile");
2051 let request = crate::control::EnsureImageRequest::new(
2052 "task",
2053 crate::testing::aarch64_daemon(b"daemon"),
2054 dockerfile,
2055 "artifact-bucket",
2056 "arn:aws:iam::123456789012:role/build",
2057 );
2058 let error = sandbox
2059 .ensure_image(request)
2060 .await
2061 .expect_err("the test adapters build no services");
2062 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
2063 assert!(
2064 error
2065 .to_string()
2066 .contains("the test adapters build no STS or S3 client for us-east-1"),
2067 "{error}"
2068 );
2069 assert!(
2070 recorder.calls().is_empty(),
2071 "no control-plane call came first"
2072 );
2073 }
2074
2075 #[tokio::test]
2083 async fn a_launch_draws_every_random_value_from_the_planes_entropy() {
2084 use crate::entropy::testing::SequenceEntropy;
2085 use base64::Engine as _;
2086
2087 let recorder = Arc::new(FakeControlPlane::new());
2088 let entropy = Arc::new(SequenceEntropy::new());
2089 let plane = crate::testing::control_plane(
2090 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2091 Region::UsEast1,
2092 Arc::new(TestClock::new()),
2093 )
2094 .with_entropy(Arc::clone(&entropy) as Arc<dyn crate::entropy::Entropy>);
2095 let mut sandbox = Sandbox::with_control_plane(plane)
2096 .with_session_backend(crate::testing::HealthyDaemon::new());
2097 answer_launch(&recorder);
2098 let session = sandbox
2099 .run(RunRequest::new().with_image("arn:image").with_identity())
2100 .await
2101 .expect("the launch reaches RUNNING");
2102
2103 assert_eq!(
2104 session.agent_token(),
2105 const_hex::encode(SequenceEntropy::draw(1, 32)),
2106 "the agent token"
2107 );
2108 let body = recorder.first_body("RunMicrovm");
2109 let payload: serde_json::Value =
2110 serde_json::from_str(body["runHookPayload"].as_str().expect("a string payload"))
2111 .expect("the payload is JSON");
2112 assert_eq!(
2113 payload[protocol::identity::SEED_KEY],
2114 base64::engine::general_purpose::STANDARD
2115 .encode(SequenceEntropy::draw(2, protocol::identity::SEED_BYTES)),
2116 "the VM seed"
2117 );
2118 let client_token = body["clientToken"].as_str().expect("a client token");
2119 assert!(
2120 client_token.ends_with(&const_hex::encode(SequenceEntropy::draw(4, 8))),
2121 "the client token's nonce: {client_token}"
2122 );
2123 assert_eq!(
2124 entropy.calls(),
2125 4,
2126 "one draw per value, the host seed included"
2127 );
2128 sandbox.detach().expect("hand the scripted VM off quietly");
2129 }
2130
2131 #[tokio::test]
2133 async fn an_unavailable_entropy_source_refuses_the_launch() {
2134 use crate::entropy::testing::SequenceEntropy;
2135
2136 let recorder = Arc::new(FakeControlPlane::new());
2137 let plane = crate::testing::control_plane(
2138 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2139 Region::UsEast1,
2140 Arc::new(TestClock::new()),
2141 )
2142 .with_entropy(Arc::new(SequenceEntropy::unavailable()));
2143 let mut sandbox = Sandbox::with_control_plane(plane)
2144 .with_session_backend(crate::testing::HealthyDaemon::new());
2145 answer_launch(&recorder);
2146 let err = sandbox
2147 .run(RunRequest::new().with_image("arn:image"))
2148 .await
2149 .expect_err("no pool, no bearer token");
2150 assert_eq!(err.kind(), ErrorKind::Unexpected, "{err}");
2151 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2152 }
2153
2154 fn launching_to(
2159 replies: Vec<crate::testing::Reply>,
2160 ) -> (
2161 Sandbox,
2162 Arc<FakeControlPlane>,
2163 Arc<crate::testing::Recorder>,
2164 ) {
2165 let recorder = Arc::new(FakeControlPlane::new());
2166 let plane = crate::testing::control_plane(
2167 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2168 Region::UsEast1,
2169 Arc::new(TestClock::new()),
2170 );
2171 let daemon = crate::testing::Recorder::with(replies);
2172 let sandbox = Sandbox::with_control_plane(plane).with_session_backend(daemon.clone());
2173 answer_launch(&recorder);
2174 (sandbox, recorder, daemon)
2175 }
2176
2177 fn not_ready_then_ready() -> Vec<crate::testing::Reply> {
2179 vec![
2180 crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()),
2181 crate::testing::Reply::ok(crate::testing::health_body(true)),
2182 ]
2183 }
2184
2185 fn daemon_paths(daemon: &crate::testing::Recorder) -> Vec<String> {
2187 daemon
2188 .requests()
2189 .iter()
2190 .map(|request| format!("{} {}", request.method, request.path))
2191 .collect()
2192 }
2193
2194 #[tokio::test(start_paused = true)]
2201 async fn a_launch_returns_once_its_daemon_answers() {
2202 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2203 sandbox
2204 .run(RunRequest::new().with_image("arn:image"))
2205 .await
2206 .expect("the launch reaches RUNNING and its daemon answers");
2207 assert_eq!(
2208 daemon_paths(&daemon),
2209 ["GET /v1/health", "GET /v1/health"],
2210 "one poll refused, one bootstrapped, and nothing after"
2211 );
2212 sandbox.detach().expect("hand the scripted VM off quietly");
2213 }
2214
2215 #[tokio::test(start_paused = true)]
2219 async fn wait_until_running_returns_once_the_daemon_answers() {
2220 let (mut sandbox, _recorder, daemon) = launching_to(not_ready_then_ready());
2221 let mut request = RunRequest::new().with_image("arn:image");
2222 request.wait = false;
2223 sandbox.run(request).await.expect("the launch is accepted");
2224 assert!(
2225 daemon_paths(&daemon).is_empty(),
2226 "an accepted launch doesn't wait"
2227 );
2228 sandbox
2229 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
2230 .await
2231 .expect("RUNNING, then the daemon answers");
2232 assert_eq!(daemon_paths(&daemon), ["GET /v1/health", "GET /v1/health"]);
2233 sandbox.detach().expect("hand the scripted VM off quietly");
2234 }
2235
2236 #[tokio::test(start_paused = true)]
2240 async fn a_launch_whose_daemon_never_answers_times_out_running() {
2241 let never = (0..128)
2242 .map(|_| crate::testing::Reply::Body(503, b"not bootstrapped".to_vec()))
2243 .collect();
2244 let (mut sandbox, _recorder, daemon) = launching_to(never);
2245 let error = sandbox
2246 .run(RunRequest::new().with_image("arn:image"))
2247 .await
2248 .expect_err("the daemon never answered");
2249 assert_eq!(error.kind(), ErrorKind::Timeout, "{error}");
2250 assert!(daemon.requests().len() > 1, "it kept asking");
2251 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2252 assert!(sandbox.token_installed());
2253 sandbox.detach().expect("hand the scripted VM off quietly");
2254 }
2255
2256 const WANTED_ARN: &str = "arn:aws:lambda:us-east-1:123456789012:microvm-image:wanted-image";
2260
2261 fn answer_listing(recorder: &FakeControlPlane) {
2263 recorder.answer(
2264 "ListMicrovmImages",
2265 Answer::ok(fake::list_images_response(
2266 &["wanted-image-old", "wanted-image"],
2267 None,
2268 )),
2269 );
2270 }
2271
2272 #[tokio::test]
2279 async fn a_launch_by_bare_image_name_sends_the_listed_arn() {
2280 let (mut sandbox, recorder, _) = planted();
2281 answer_listing(&recorder);
2282 answer_launch(&recorder);
2283 sandbox
2284 .run(RunRequest::new().with_image("wanted-image"))
2285 .await
2286 .expect("the launch reaches RUNNING");
2287
2288 let operations = recorder.operations();
2289 assert_eq!(
2290 operations.get(..2),
2291 Some(&["ListMicrovmImages", "RunMicrovm"][..]),
2292 "the name is resolved, once, before the launch: {operations:?}"
2293 );
2294 assert_eq!(recorder.call_count("ListMicrovmImages"), 1);
2295 assert_eq!(
2296 recorder.first_body("RunMicrovm")["imageIdentifier"],
2297 WANTED_ARN,
2298 "the launch carries the exact match's ARN, never the name"
2299 );
2300 sandbox.detach().expect("hand the scripted VM off quietly");
2301 }
2302
2303 #[tokio::test]
2305 async fn a_launch_by_image_arn_makes_no_listing_call() {
2306 let (mut sandbox, recorder, _) = planted();
2307 answer_launch(&recorder);
2308 sandbox
2309 .run(RunRequest::new().with_image(WANTED_ARN))
2310 .await
2311 .expect("the launch reaches RUNNING");
2312
2313 assert_eq!(recorder.call_count("ListMicrovmImages"), 0);
2314 assert_eq!(recorder.operations().first(), Some(&"RunMicrovm"));
2315 assert_eq!(
2316 recorder.first_body("RunMicrovm")["imageIdentifier"],
2317 WANTED_ARN
2318 );
2319 assert_eq!(sandbox.launch_image_arn(), Some(WANTED_ARN));
2320 sandbox.detach().expect("hand the scripted VM off quietly");
2321 }
2322
2323 #[tokio::test]
2330 async fn a_refused_launch_by_name_still_records_the_resolved_arn() {
2331 let (mut sandbox, recorder, _) = planted();
2332 answer_listing(&recorder);
2333 recorder.answer("RunMicrovm", Answer::failure(400, "scripted refusal"));
2334 sandbox
2335 .run(RunRequest::new().with_image("wanted-image"))
2336 .await
2337 .expect_err("the service refused the launch");
2338
2339 assert_eq!(
2340 sandbox.launch_image_arn(),
2341 Some(WANTED_ARN),
2342 "the ARN the refused launch asked for"
2343 );
2344 assert!(sandbox.microvm().is_none(), "no launch was accepted");
2345 assert_eq!(recorder.call_count("RunMicrovm"), 1);
2346 }
2347
2348 #[tokio::test]
2352 async fn an_unknown_image_name_is_refused_before_the_launch_and_the_sandbox_still_runs() {
2353 let (mut sandbox, recorder, _) = planted();
2354 recorder.answer(
2355 "ListMicrovmImages",
2356 Answer::ok(fake::list_images_response(&[], None)),
2357 );
2358 let error = sandbox
2359 .run(RunRequest::new().with_image("no-such-image"))
2360 .await
2361 .expect_err("nothing to launch from");
2362 assert_eq!(error.code(), "ERR_PRECONDITION");
2363 assert!(error.to_string().contains("no-such-image"), "{error}");
2364 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2365 assert_eq!(sandbox.launch_image_arn(), None);
2366 assert!(!sandbox.image_exists(), "STATE-1: no launch was accepted");
2367 assert!(sandbox.microvm().is_none());
2368 assert_eq!(sandbox.bootstrap_count(), 0);
2369
2370 answer_launch(&recorder);
2371 sandbox
2372 .run(RunRequest::new().with_image(WANTED_ARN))
2373 .await
2374 .expect("a failed resolution doesn't use up the sandbox's one launch");
2375 assert_eq!(sandbox.bootstrap_count(), 1);
2376 sandbox.detach().expect("hand the scripted VM off quietly");
2377 }
2378
2379 #[tokio::test]
2385 async fn a_launch_refused_locally_makes_no_listing_call_for_a_name() {
2386 let (mut sandbox, recorder, _) = planted();
2387 answer_listing(&recorder);
2388 let mut request = RunRequest::new().with_image("wanted-image");
2389 request.max_duration_sec = 0;
2390 let error = sandbox
2391 .run(request)
2392 .await
2393 .expect_err("a zero maximum duration is refused locally");
2394 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
2395 assert!(
2396 recorder.calls().is_empty(),
2397 "the refusal came after a call: {:?}",
2398 recorder.operations()
2399 );
2400 assert_eq!(sandbox.launch_image_arn(), None);
2401 }
2402
2403 #[tokio::test]
2410 async fn a_launched_session_reports_its_requests_posture() {
2411 use crate::control::EgressPosture;
2412 let arn = "arn:aws:lambda:us-east-1:123456789012:network-connector:isolated-vpc";
2413 let rows = [
2414 (RunRequest::new(), EgressPosture::Unsealed),
2415 (RunRequest::new().with_egress(), EgressPosture::Open),
2416 (
2417 RunRequest::new().with_deny_egress(),
2418 EgressPosture::BestEffort,
2419 ),
2420 (
2421 RunRequest::new().with_egress_network_connector(arn),
2422 EgressPosture::Unsealed,
2423 ),
2424 ];
2425 for (request, expected) in rows {
2426 let answered = crate::control::egress_posture_for(
2427 request.egress,
2428 &request.egress_network_connectors,
2429 request.deny_egress,
2430 Some(&Region::UsEast1),
2431 )
2432 .expect("a launchable request");
2433 assert_eq!(answered, expected);
2434 let (mut sandbox, recorder, _) = planted();
2435 answer_launch(&recorder);
2436 let session = sandbox
2437 .run(request.with_image("arn:image"))
2438 .await
2439 .expect("the launch reaches RUNNING");
2440 assert_eq!(session.egress_posture(), expected, "BIND-12");
2441 assert_eq!(sandbox.egress_posture(), expected);
2442 sandbox.detach().expect("hand the scripted VM off quietly");
2443 }
2444 }
2445
2446 #[tokio::test]
2450 async fn a_session_without_its_launch_options_reports_unsealed() {
2451 use crate::control::EgressPosture;
2452 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2453 assert_eq!(
2454 sandbox.session().expect("RUNNING").egress_posture(),
2455 EgressPosture::Unsealed
2456 );
2457 assert_eq!(sandbox.egress_posture(), EgressPosture::Unsealed);
2458 let direct = Session::builder(ADOPT_ENDPOINT, ADOPT_TOKEN)
2459 .build_with(&crate::testing::TestAdapters::new())
2460 .expect("a direct session");
2461 assert_eq!(direct.egress_posture(), EgressPosture::Unsealed);
2462 }
2463
2464 async fn launched() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2466 let (mut sandbox, recorder, clock) = planted();
2467 answer_launch(&recorder);
2468 sandbox
2469 .run(RunRequest::new().with_image("arn:image"))
2470 .await
2471 .expect("the launch reaches RUNNING");
2472 (sandbox, recorder, clock)
2473 }
2474
2475 #[tokio::test]
2481 async fn a_dropped_live_sandbox_warns_through_the_planes_adapters() {
2482 let recorder = Arc::new(FakeControlPlane::new());
2483 let clock = Arc::new(TestClock::new());
2484 let adapters = Arc::new(crate::testing::TestAdapters::new());
2485 let plane = crate::control::ControlPlane::from_ports(
2486 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2487 Region::UsEast1,
2488 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2489 Arc::new(crate::testing::SequenceEntropy::new()),
2490 Arc::clone(&adapters) as Arc<dyn crate::adapters::Adapters>,
2491 );
2492 let mut sandbox = Sandbox::with_control_plane(plane)
2493 .with_session_backend(crate::testing::HealthyDaemon::new());
2494 answer_launch(&recorder);
2495 sandbox
2496 .run(RunRequest::new().with_image("arn:image"))
2497 .await
2498 .expect("the launch reaches RUNNING");
2499 let id = sandbox.microvm().expect("launched").id.clone();
2500 drop(sandbox);
2501 let warnings = adapters.warnings();
2502 assert_eq!(warnings.len(), 1, "{warnings:?}");
2503 assert!(warnings[0].contains(&id), "{warnings:?}");
2504 }
2505
2506 #[tokio::test]
2516 async fn a_detached_sandbox_hands_off_its_vm_and_stays_quiet() {
2517 let (mut sandbox, recorder, _clock) = launched().await;
2518 assert!(
2519 sandbox.drop_warning().is_some(),
2520 "a launched, live VM warns on drop"
2521 );
2522 let token = sandbox
2523 .session()
2524 .expect("launched")
2525 .agent_token()
2526 .to_string();
2527 let vm = sandbox.microvm().expect("launched").clone();
2528
2529 let detached = sandbox.detach().expect("a RUNNING VM detaches");
2530 assert_eq!(detached.microvm_id, vm.id);
2531 assert_eq!(detached.endpoint, vm.endpoint);
2532 assert_eq!(detached.agent_token(), token);
2533 assert_eq!(detached.region, Region::UsEast1);
2534 let printed = format!("{detached:?} {sandbox:?}");
2535 assert!(!printed.contains(&token), "{printed}");
2536 assert!(printed.contains("detached: true"), "{printed}");
2537
2538 assert!(sandbox.detached());
2539 assert!(
2540 sandbox.drop_warning().is_none(),
2541 "detached is not abandoned"
2542 );
2543 assert!(
2544 sandbox.session().is_none(),
2545 "the session went to the adopter"
2546 );
2547
2548 let before = recorder.calls().len();
2549 for error in [
2550 sandbox.suspend().await.expect_err("detached"),
2551 sandbox.resume().await.map(|_| ()).expect_err("detached"),
2552 sandbox
2553 .run(RunRequest::new().with_image("arn:image"))
2554 .await
2555 .map(|_| ())
2556 .expect_err("detached"),
2557 sandbox
2558 .wait_until_running(Duration::from_secs(1))
2559 .await
2560 .map(|_| ())
2561 .expect_err("detached"),
2562 sandbox.detach().map(|_| ()).expect_err("already detached"),
2563 ] {
2564 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2565 assert!(error.to_string().contains("detached"), "{error}");
2566 assert!(!error.to_string().contains(&token), "{error}");
2567 }
2568 let report = sandbox.terminate(TeardownOpts::default()).await;
2569 assert!(!report.terminate_accepted);
2570 assert!(report.failures[0].contains("detached"), "{report:?}");
2571 assert_eq!(recorder.calls().len(), before, "detached makes no AWS call");
2572 assert!(
2573 sandbox.drop_warning().is_none(),
2574 "a refused terminate is still quiet"
2575 );
2576 }
2577
2578 #[tokio::test]
2580 async fn detach_refuses_without_a_live_vm() {
2581 let (mut unlaunched, _recorder, _clock) = planted();
2582 let error = unlaunched.detach().expect_err("nothing launched");
2583 assert_eq!(error.kind(), ErrorKind::Precondition);
2584 assert!(!unlaunched.detached());
2585
2586 let (mut sandbox, recorder, _clock) = launched().await;
2587 recorder
2588 .answer("TerminateMicrovm", Answer::ok("{}"))
2589 .answer(
2590 "GetMicrovm",
2591 Answer::ok(fake::microvm_response("TERMINATED", None)),
2592 );
2593 sandbox.terminate(TeardownOpts::default()).await;
2594 let error = sandbox.detach().expect_err("torn down");
2595 assert_eq!(error.kind(), ErrorKind::Precondition, "{error}");
2596 assert!(!sandbox.detached());
2597 }
2598
2599 async fn suspended_with_window(
2610 suspended_sec: u32,
2611 ) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2612 let (mut sandbox, recorder, clock) = planted();
2613 recorder
2614 .answer(
2615 "RunMicrovm",
2616 Answer::ok(fake::microvm_response("PENDING", None)),
2617 )
2618 .answer(
2619 "GetMicrovm",
2620 Answer::ok(fake::microvm_response("RUNNING", None)),
2621 )
2622 .answer(
2623 "GetMicrovm",
2624 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2625 )
2626 .answer(
2627 "CreateMicrovmAuthToken",
2628 Answer::ok(fake::auth_token_response("proxy-token")),
2629 )
2630 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2631
2632 sandbox
2633 .run(
2634 RunRequest::new()
2635 .with_image("arn:image")
2636 .with_suspended_sec(suspended_sec),
2637 )
2638 .await
2639 .expect("launches");
2640 sandbox.suspend().await.expect("suspends");
2641 assert_eq!(
2642 clock.now(),
2643 Duration::ZERO,
2644 "no wait may have slept, or the window arithmetic below is measuring the fake's \
2645 poll interval as well as the elapsed suspension"
2646 );
2647 (sandbox, recorder, clock)
2648 }
2649
2650 async fn suspended() -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2652 suspended_with_window(600).await
2653 }
2654
2655 const ADOPT_TOKEN: &str = "adopt-canary-token-9f1c";
2659 const ADOPT_ENDPOINT: &str = "https://mvm-abc123.microvm.us-east-1.amazonaws.com";
2660
2661 fn adopt_plane() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
2663 let recorder = Arc::new(FakeControlPlane::new());
2664 let clock = Arc::new(TestClock::new());
2665 let plane = crate::testing::control_plane(
2666 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
2667 Region::UsEast1,
2668 Arc::clone(&clock) as Arc<dyn crate::control::Clock>,
2669 );
2670 (plane, recorder, clock)
2671 }
2672
2673 async fn adopted_in(state: &str) -> (Sandbox, Arc<FakeControlPlane>, Arc<TestClock>) {
2674 let (plane, recorder, clock) = adopt_plane();
2675 recorder.answer(
2676 "GetMicrovm",
2677 Answer::ok(fake::microvm_response(state, None)),
2678 );
2679 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2680 .await
2681 .expect("adopts");
2682 (sandbox, recorder, clock)
2683 }
2684
2685 #[tokio::test]
2692 async fn an_adopted_sandbox_reports_the_services_idle_window_and_watches_its_lifecycle() {
2693 let (sandbox, _recorder, _clock) = adopted_in("RUNNING").await;
2694 assert_eq!(sandbox.idle_window(), Some(Duration::from_secs(1800)));
2695 assert_eq!(*sandbox.watch_lifecycle().borrow(), Lifecycle::Running);
2696 }
2697
2698 #[test]
2700 fn every_service_state_maps_onto_one_lifecycle() {
2701 for state in crate::constants::MICROVM_STATES {
2702 let lifecycle = Lifecycle::from_service(state).expect("a known state");
2703 assert_eq!(lifecycle.as_str(), state);
2704 }
2705 assert_eq!(Lifecycle::from_service("Running"), None);
2706 }
2707
2708 #[tokio::test]
2716 async fn an_adopted_vm_takes_its_lifecycle_from_the_service_and_refuses_run() {
2717 let (mut sandbox, recorder, _) = adopted_in("RUNNING").await;
2718 assert!(sandbox.adopted());
2719 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2720 assert!(sandbox.token_installed());
2721 assert_eq!(sandbox.bootstrap_count(), 1);
2722 assert!(sandbox.image_exists());
2723 let session = sandbox.session().expect("a live VM gets a session");
2724 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2725 assert_eq!(session.agent_token(), ADOPT_TOKEN);
2726
2727 let error = sandbox
2728 .run(RunRequest::new().with_image("arn:image"))
2729 .await
2730 .expect_err("an adopted VM was bootstrapped by its own launch");
2731 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2732 assert!(error.to_string().contains("STATE-3"), "{error}");
2733 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2734 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2735 }
2736
2737 #[tokio::test]
2743 async fn a_name_adopts_its_records_vm_and_foreign_names_are_refused_locally() {
2744 struct One(crate::names::NameRecord);
2745 impl crate::names::NameStore for One {
2746 fn get(&self, name: &str) -> Result<Option<crate::names::NameRecord>, Error> {
2747 Ok((name == self.0.name).then(|| self.0.clone()))
2748 }
2749 fn put(&self, _: &crate::names::NameRecord) -> Result<(), Error> {
2750 unreachable!()
2751 }
2752 fn delete(&self, _: &str) -> Result<bool, Error> {
2753 unreachable!()
2754 }
2755 fn list(&self) -> Result<Vec<crate::names::NameRecord>, Error> {
2756 Ok(vec![self.0.clone()])
2757 }
2758 }
2759 let record = crate::names::NameRecord::new_at(
2760 "ci",
2761 "mvm-abc123",
2762 ADOPT_ENDPOINT,
2763 ADOPT_TOKEN,
2764 "us-west-2",
2765 1_767_225_600,
2766 )
2767 .expect("record");
2768 let store = One(record.clone());
2769 let foreign = crate::names::resolve(&store, "ci", Some(&Region::UsEast1))
2770 .expect_err("registered in another region");
2771 assert_eq!(foreign.kind(), ErrorKind::InvalidArg);
2772 let missing = crate::names::resolve(&store, "nope", None).expect_err("no such name");
2773 assert_eq!(missing.kind(), ErrorKind::Precondition);
2774 assert!(!format!("{foreign} {missing}").contains(ADOPT_TOKEN));
2775
2776 let (plane, recorder, _) = adopt_plane();
2777 recorder.answer(
2778 "GetMicrovm",
2779 Answer::ok(fake::microvm_response("RUNNING", None)),
2780 );
2781 let resolved = crate::names::resolve(&store, "ci", Some(&Region::UsWest2)).expect("ok");
2782 let sandbox = Sandbox::adopt_record(plane, resolved)
2783 .await
2784 .expect("adopts");
2785 assert!(sandbox.adopted());
2786 assert_eq!(
2787 sandbox.microvm().map(|vm| vm.id.as_str()),
2788 Some("mvm-abc123")
2789 );
2790 assert_eq!(
2791 sandbox.session().expect("session").agent_token(),
2792 ADOPT_TOKEN
2793 );
2794 let named = sandbox
2795 .name_record("again")
2796 .expect("an adopted VM can be named");
2797 assert_eq!(
2798 (
2799 named.microvm_id.as_str(),
2800 named.agent_token.as_str(),
2801 named.region.as_str()
2802 ),
2803 ("mvm-abc123", ADOPT_TOKEN, "us-east-1")
2804 );
2805 let unlaunched = Sandbox::with_control_plane(adopt_plane().0);
2806 assert_eq!(
2807 unlaunched
2808 .name_record("x")
2809 .expect_err("nothing to name")
2810 .kind(),
2811 ErrorKind::Precondition
2812 );
2813 }
2814
2815 #[tokio::test]
2821 async fn a_records_tunnel_identity_survives_adoption_and_naming() {
2822 let identity = crate::identity::LaunchIdentity::from_seeds([7; 32], [9; 32])
2823 .expect("valid seeds")
2824 .keep();
2825 let mut record = crate::names::NameRecord::new_at(
2826 "ci",
2827 "mvm-abc123",
2828 ADOPT_ENDPOINT,
2829 ADOPT_TOKEN,
2830 "us-east-1",
2831 1_767_225_600,
2832 )
2833 .expect("record");
2834 record.identity_host_seed = Some(identity.host_seed_base64());
2835 record.identity_vm_public_key = Some(identity.vm_public_key_base64());
2836
2837 let (plane, recorder, _) = adopt_plane();
2838 recorder.answer(
2839 "GetMicrovm",
2840 Answer::ok(fake::microvm_response("RUNNING", None)),
2841 );
2842 let sandbox = Sandbox::adopt_record(plane, record.clone())
2843 .await
2844 .expect("adopts");
2845 let kept = sandbox
2846 .tunnel_identity()
2847 .expect("the record's pair is kept");
2848 assert_eq!(kept.host_seed_base64(), identity.host_seed_base64());
2849 assert_eq!(kept.vm_public_key_base64(), identity.vm_public_key_base64());
2850 let named = sandbox.name_record("again").expect("names");
2851 assert_eq!(named.identity_host_seed, record.identity_host_seed);
2852 assert_eq!(named.identity_vm_public_key, record.identity_vm_public_key);
2853
2854 let (plane, recorder, _) = adopt_plane();
2855 let mut torn = record;
2856 torn.identity_vm_public_key = None;
2857 let refused = Sandbox::adopt_record(plane, torn)
2858 .await
2859 .expect_err("half a pair is refused");
2860 assert_eq!(refused.kind(), ErrorKind::InvalidArg);
2861 assert!(refused.to_string().contains("one half"), "{refused}");
2862 assert!(recorder.calls().is_empty(), "refused before any AWS call");
2863 }
2864
2865 #[tokio::test]
2868 async fn an_adopted_vm_suspends_resumes_and_terminates_through_its_handle() {
2869 let (plane, recorder, _) = adopt_plane();
2870 recorder
2871 .answer(
2872 "GetMicrovm",
2873 Answer::ok(fake::microvm_response("RUNNING", None)),
2874 )
2875 .answer(
2876 "GetMicrovm",
2877 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2878 )
2879 .answer(
2880 "GetMicrovm",
2881 Answer::ok(fake::microvm_response("RUNNING", None)),
2882 )
2883 .answer(
2884 "GetMicrovm",
2885 Answer::ok(fake::microvm_response("TERMINATED", None)),
2886 )
2887 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
2888 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2889 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
2890 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2891 .await
2892 .expect("adopts");
2893
2894 sandbox.suspend().await.expect("suspends from RUNNING");
2895 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2896 sandbox.resume().await.expect("resumes from SUSPENDED");
2897 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
2898 let report = sandbox
2899 .terminate(TeardownOpts::default().waiting_for_terminated())
2900 .await;
2901 assert!(report.terminate_accepted, "{report:?}");
2902 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
2903 assert!(sandbox.was_terminated());
2904
2905 assert_eq!(recorder.call_count("RunMicrovm"), 0);
2906 assert!(
2907 recorder
2908 .bodies_as_text()
2909 .iter()
2910 .all(|body| !body.contains("runHookPayload") && !body.contains(ADOPT_TOKEN)),
2911 "no launch payload or agent token may reach the control plane"
2912 );
2913 assert_eq!(sandbox.bootstrap_count(), 1, "STATE-3: never above one");
2914 }
2915
2916 #[tokio::test]
2923 async fn an_adopted_vm_refuses_a_resume_past_the_reported_window() {
2924 let (plane, recorder, clock) = adopt_plane();
2925 recorder
2926 .answer(
2927 "GetMicrovm",
2928 Answer::ok(fake::microvm_response("RUNNING", None)),
2929 )
2930 .answer(
2931 "GetMicrovm",
2932 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2933 )
2934 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
2935 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2936 .await
2937 .expect("adopts");
2938 assert_eq!(sandbox.suspended_window(), None, "no request of its own");
2939 sandbox.suspend().await.expect("suspends");
2940
2941 clock.advance(Duration::from_secs(601));
2942 let error = sandbox
2943 .resume()
2944 .await
2945 .expect_err("past the 600 s window the service reported");
2946 assert_eq!(error.kind(), ErrorKind::WindowClosed);
2947 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
2948 }
2949
2950 #[tokio::test]
2953 async fn an_adopted_suspended_vm_resumes_through_the_service() {
2954 let (plane, recorder, _) = adopt_plane();
2955 recorder
2956 .answer(
2957 "GetMicrovm",
2958 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2959 )
2960 .answer(
2961 "GetMicrovm",
2962 Answer::ok(fake::microvm_response("RUNNING", None)),
2963 )
2964 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
2965 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
2966 .await
2967 .expect("adopts");
2968 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
2969 let session = sandbox
2970 .resume()
2971 .await
2972 .expect("the service answers the resume");
2973 assert_eq!(session.endpoint(), ADOPT_ENDPOINT);
2974 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
2975 }
2976
2977 #[tokio::test]
2980 async fn an_adopted_terminated_vm_refuses_every_transition_locally() {
2981 let (mut sandbox, recorder, _) = adopted_in("TERMINATED").await;
2982 assert!(sandbox.was_terminated());
2983 assert!(sandbox.session().is_none());
2984 let resume = sandbox.resume().await.expect_err("STATE-11");
2985 assert!(resume.to_string().contains("STATE-11"), "{resume}");
2986 let suspend = sandbox.suspend().await.expect_err("STATE-5");
2987 assert!(suspend.to_string().contains("STATE-5"), "{suspend}");
2988 assert_eq!(recorder.operations(), vec!["GetMicrovm"]);
2989 }
2990
2991 async fn adopted_with_a_daemon(
2994 state: &str,
2995 ) -> (
2996 Sandbox,
2997 Arc<FakeControlPlane>,
2998 Arc<crate::testing::HealthyDaemon>,
2999 ) {
3000 let recorder = Arc::new(FakeControlPlane::new());
3001 let daemon = crate::testing::HealthyDaemon::new();
3002 let plane = crate::control::ControlPlane::from_ports(
3003 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3004 Region::UsEast1,
3005 Arc::new(TestClock::new()),
3006 Arc::new(crate::testing::SequenceEntropy::new()),
3007 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3008 );
3009 recorder
3010 .answer(
3011 "GetMicrovm",
3012 Answer::ok(fake::microvm_response(state, None)),
3013 )
3014 .answer(
3015 "GetMicrovm",
3016 Answer::ok(fake::microvm_response("RUNNING", None)),
3017 )
3018 .answer(
3019 "CreateMicrovmAuthToken",
3020 Answer::ok(fake::auth_token_response("proxy-token")),
3021 );
3022 let sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3023 .await
3024 .expect("adopts");
3025 (sandbox, recorder, daemon)
3026 }
3027
3028 #[tokio::test]
3035 async fn wait_until_ready_finishes_whichever_wait_is_left() {
3036 let (mut pending, recorder, daemon) = adopted_with_a_daemon("PENDING").await;
3037 pending
3038 .wait_until_ready(Duration::from_secs(60))
3039 .await
3040 .expect("RUNNING, then the daemon");
3041 assert_eq!(pending.lifecycle(), Lifecycle::Running);
3042 assert_eq!(pending.bootstrap_count(), 1);
3043 assert_eq!(daemon.polls(), 1);
3044 assert_eq!(
3045 recorder.call_count("GetMicrovm"),
3046 2,
3047 "the adopt's read and the wait's"
3048 );
3049
3050 let (mut running, recorder, daemon) = adopted_with_a_daemon("RUNNING").await;
3051 running
3052 .wait_until_ready(Duration::from_secs(60))
3053 .await
3054 .expect("the daemon answers");
3055 assert_eq!(daemon.polls(), 1);
3056 assert_eq!(
3057 recorder.call_count("GetMicrovm"),
3058 1,
3059 "RUNNING needs no second read"
3060 );
3061 }
3062
3063 #[tokio::test]
3065 async fn wait_until_ready_refuses_a_suspended_vm() {
3066 let (mut suspended, _, daemon) = adopted_with_a_daemon("SUSPENDED").await;
3067 let error = suspended
3068 .wait_until_ready(Duration::from_secs(60))
3069 .await
3070 .expect_err("a suspended VM isn't starting");
3071 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3072 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3073 assert_eq!(daemon.polls(), 0);
3074 }
3075
3076 #[tokio::test]
3079 async fn an_adopted_pending_vm_counts_its_bootstrap_once_on_running() {
3080 let recorder = Arc::new(FakeControlPlane::new());
3081 let daemon = crate::testing::HealthyDaemon::new();
3082 let plane = crate::control::ControlPlane::from_ports(
3083 Arc::clone(&recorder) as Arc<dyn crate::control::transport::Transport>,
3084 Region::UsEast1,
3085 Arc::new(TestClock::new()),
3086 Arc::new(crate::testing::SequenceEntropy::new()),
3087 Arc::new(crate::testing::TestAdapters::new().with_backend(daemon.clone())),
3088 );
3089 recorder
3090 .answer(
3091 "GetMicrovm",
3092 Answer::ok(fake::microvm_response("PENDING", None)),
3093 )
3094 .answer(
3095 "GetMicrovm",
3096 Answer::ok(fake::microvm_response("RUNNING", None)),
3097 )
3098 .answer(
3099 "CreateMicrovmAuthToken",
3100 Answer::ok(fake::auth_token_response("proxy-token")),
3101 );
3102 let mut sandbox = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3103 .await
3104 .expect("adopts");
3105 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3106 assert_eq!(sandbox.bootstrap_count(), 0);
3107 sandbox
3108 .wait_until_running(Duration::from_secs(60))
3109 .await
3110 .expect("reaches RUNNING");
3111 assert_eq!(sandbox.bootstrap_count(), 1);
3112 assert!(sandbox.token_installed());
3113 assert_eq!(
3114 daemon.polls(),
3115 1,
3116 "the daemon answered before the session came back"
3117 );
3118 assert!(
3119 sandbox
3120 .run(RunRequest::new().with_image("arn:image"))
3121 .await
3122 .is_err(),
3123 "still one bootstrap per VM"
3124 );
3125 assert_eq!(recorder.call_count("RunMicrovm"), 0);
3126 }
3127
3128 #[tokio::test]
3135 async fn adopt_refuses_bad_records_and_never_prints_the_token() {
3136 let (plane, recorder, _) = adopt_plane();
3137 let empty = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, "")
3138 .await
3139 .expect_err("no token");
3140 assert_eq!(empty.kind(), ErrorKind::InvalidArg);
3141 assert_eq!(recorder.calls().len(), 0, "refused before any call");
3142
3143 let (plane, recorder, _) = adopt_plane();
3144 recorder.answer(
3145 "GetMicrovm",
3146 Answer::ok(fake::microvm_response("RUNNING", None)),
3147 );
3148 let mismatch = Sandbox::adopt(plane, "mvm-abc123", "https://other.example", ADOPT_TOKEN)
3149 .await
3150 .expect_err("the endpoint belongs to another record");
3151 assert_eq!(mismatch.kind(), ErrorKind::InvalidArg);
3152 assert!(!mismatch.to_string().contains(ADOPT_TOKEN));
3153
3154 let (plane, recorder, _) = adopt_plane();
3155 recorder.answer(
3156 "GetMicrovm",
3157 Answer::ok(fake::microvm_response("REBOOTING", None)),
3158 );
3159 let unknown = Sandbox::adopt(plane, "mvm-abc123", ADOPT_ENDPOINT, ADOPT_TOKEN)
3160 .await
3161 .expect_err("an unknown state");
3162 assert_eq!(unknown.kind(), ErrorKind::Platform);
3163 assert!(!unknown.to_string().contains(ADOPT_TOKEN));
3164
3165 let (sandbox, _, _) = adopted_in("RUNNING").await;
3166 let printed = format!("{sandbox:?}");
3167 assert!(!printed.contains(ADOPT_TOKEN), "{printed}");
3168 assert!(printed.contains("adopted: true"), "{printed}");
3169 }
3170
3171 #[tokio::test]
3181 async fn a_launch_records_the_image_and_the_running_report_installs_the_token() {
3182 let (mut sandbox, recorder, _) = planted();
3183 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3184 assert!(
3185 !sandbox.image_exists(),
3186 "nothing is recorded before a launch"
3187 );
3188 assert!(!sandbox.token_installed());
3189
3190 answer_launch(&recorder);
3191 let session = sandbox
3192 .run(RunRequest::new().with_image("arn:image"))
3193 .await
3194 .expect("launches");
3195 assert_eq!(
3196 session.endpoint(),
3197 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3198 );
3199
3200 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3201 assert!(sandbox.image_exists(), "STATE-1: the image is recorded");
3202 assert!(sandbox.token_installed(), "STATE-2: the token is installed");
3203 assert_eq!(sandbox.bootstrap_count(), 1);
3204 assert_eq!(sandbox.microvm().expect("launched").id, "mvm-abc123");
3205
3206 let body = recorder.first_body("RunMicrovm");
3209 let payload = body["runHookPayload"].as_str().expect("a string");
3210 assert!(payload.starts_with(r#"{"agent_token":"#), "{payload}");
3211 assert_eq!(body["idlePolicy"]["suspendedDurationSeconds"], 600);
3212 }
3213
3214 #[tokio::test]
3221 async fn a_launch_env_reaches_the_run_hook_payload_on_the_wire() {
3222 let (mut sandbox, recorder, _) = planted();
3223 answer_launch(&recorder);
3224 sandbox
3225 .run(
3226 RunRequest::new()
3227 .with_image("arn:image")
3228 .with_launch_env("ANTHROPIC_BASE_URL", "https://gateway.example")
3229 .with_launch_env("PATH", "/usr/local/bin:/usr/bin:/bin"),
3230 )
3231 .await
3232 .expect("launches");
3233
3234 let body = recorder.first_body("RunMicrovm");
3235 let payload = body["runHookPayload"].as_str().expect("a string");
3236 let inner: serde_json::Value =
3238 serde_json::from_str(payload).expect("the payload is itself JSON");
3239 assert!(
3240 inner["agent_token"].as_str().is_some_and(|t| !t.is_empty()),
3241 "the token still rides alongside: {payload}"
3242 );
3243 assert_eq!(
3244 inner["env"]["ANTHROPIC_BASE_URL"],
3245 "https://gateway.example"
3246 );
3247 assert_eq!(inner["env"]["PATH"], "/usr/local/bin:/usr/bin:/bin");
3248 }
3249
3250 #[tokio::test]
3263 async fn the_advisory_deny_reaches_the_launch_environment_in_both_spellings() {
3264 let (mut sandbox, recorder, _) = planted();
3265 answer_launch(&recorder);
3266 let request = RunRequest::new().with_image("arn:image").with_deny_egress();
3267 assert_eq!(
3268 request.egress_posture(),
3269 crate::control::EgressPosture::BestEffort
3270 );
3271 sandbox.run(request).await.expect("launches");
3272
3273 let body = recorder.first_body("RunMicrovm");
3274 let inner: serde_json::Value =
3275 serde_json::from_str(body["runHookPayload"].as_str().expect("a string"))
3276 .expect("the payload is itself JSON");
3277 for key in crate::sandbox::DENY_EGRESS_ENV_KEYS {
3278 assert_eq!(
3279 inner["env"][key], DENY_EGRESS_PROXY_URL,
3280 "{key} must point at the black hole: {inner}"
3281 );
3282 }
3283 assert!(
3284 body.get("egressNetworkConnectors").is_none(),
3285 "the advisory deny asks the platform for nothing: {body}"
3286 );
3287 }
3288
3289 #[test]
3292 fn the_deny_never_overwrites_a_callers_own_proxy_and_is_absent_by_default() {
3293 let kept = RunRequest::new()
3294 .with_deny_egress()
3295 .with_launch_env("https_proxy", "http://proxy.internal:3128")
3296 .effective_launch_env();
3297 assert_eq!(kept["https_proxy"], "http://proxy.internal:3128");
3298 assert_eq!(kept["http_proxy"], DENY_EGRESS_PROXY_URL);
3299
3300 let plain = RunRequest::new().effective_launch_env();
3301 assert!(
3302 plain.is_empty(),
3303 "a launch that did not ask for the deny sends byte-for-byte what it always sent: \
3304 {plain:?}"
3305 );
3306 assert_eq!(
3307 RunRequest::new().egress_posture(),
3308 crate::control::EgressPosture::Unsealed,
3309 "and its posture is the measured one, not a seal"
3310 );
3311 }
3312
3313 #[tokio::test]
3322 async fn asking_for_egress_and_the_deny_together_is_refused_before_any_call() {
3323 let (mut sandbox, recorder, _) = planted();
3324 answer_launch(&recorder);
3325 let mut request = RunRequest::new().with_image("arn:image").with_egress();
3326 request.deny_egress = true;
3327
3328 let error = sandbox
3329 .run(request)
3330 .await
3331 .expect_err("opposite intents are not resolved by a precedence rule");
3332 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3333 assert!(error.to_string().contains("opposite things"), "{error}");
3334 assert_eq!(
3335 recorder.calls().len(),
3336 0,
3337 "a locally-refused launch costs nothing"
3338 );
3339 }
3340
3341 #[tokio::test]
3352 async fn a_pinned_image_version_is_forwarded_to_the_launch_request() {
3353 let (mut sandbox, recorder, _) = planted();
3354 answer_launch(&recorder);
3355 sandbox
3356 .run(
3357 RunRequest::new()
3358 .with_image("arn:image")
3359 .with_image_version("2.0"),
3360 )
3361 .await
3362 .expect("launches");
3363 assert_eq!(
3364 recorder.first_body("RunMicrovm")["imageVersion"],
3365 "2.0",
3366 "a canary has to launch against the version it means to test"
3367 );
3368
3369 let (mut sandbox, recorder, _) = planted();
3370 answer_launch(&recorder);
3371 sandbox
3372 .run(RunRequest::new().with_image("arn:image"))
3373 .await
3374 .expect("launches");
3375 assert!(
3376 recorder
3377 .first_body("RunMicrovm")
3378 .get("imageVersion")
3379 .is_none(),
3380 "an unpinned launch sends what this client always sent"
3381 );
3382 }
3383
3384 #[tokio::test]
3395 async fn an_over_budget_launch_env_is_refused_before_any_control_plane_call() {
3396 let (mut sandbox, recorder, _) = planted();
3397 answer_launch(&recorder);
3398
3399 let error = sandbox
3400 .run(
3401 RunRequest::new()
3402 .with_image("arn:image")
3403 .with_launch_env("AWS_SESSION_TOKEN", "t".repeat(4096)),
3404 )
3405 .await
3406 .expect_err("a credential-scale launch env does not fit the payload");
3407
3408 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3409 let message = error.to_string();
3410 assert!(message.contains("ceiling of 4096"), "{message}");
3411 assert!(message.contains("launch env contributed"), "{message}");
3412 assert_eq!(
3413 recorder.calls().len(),
3414 0,
3415 "the refusal has to be local: a launch that reached AWS costs a VM and a bill"
3416 );
3417 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3420 assert_eq!(sandbox.bootstrap_count(), 0);
3421 assert!(sandbox.microvm().is_none());
3422 }
3423
3424 #[tokio::test]
3442 async fn the_bindings_layer_refuses_an_illegal_launch_with_zero_calls() {
3443 let cases: [(&str, RunRequest, &str); 3] = [
3444 (
3445 "max_idle_sec=59, which is what a Python keyword argument passes",
3446 RunRequest {
3447 max_idle_sec: 59,
3448 ..RunRequest::new().with_image("arn:image")
3449 },
3450 "maxIdleDurationSeconds",
3451 ),
3452 (
3453 "an execution role that is a bare name",
3454 RunRequest {
3455 execution_role_arn: Some("execution-role".to_string()),
3456 ..RunRequest::new().with_image("arn:image")
3457 },
3458 "role *name*",
3459 ),
3460 (
3461 "a pinned version with a trailing newline",
3462 RunRequest::new()
3463 .with_image("arn:image")
3464 .with_image_version("2.0\n"),
3465 "contains whitespace",
3466 ),
3467 ];
3468
3469 for (label, request, expected) in cases {
3470 let (mut sandbox, recorder, _) = planted();
3471 answer_launch(&recorder);
3474
3475 let Err(error) = sandbox.run(request).await else {
3479 panic!("{label} must be refused");
3480 };
3481 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
3482 assert!(
3483 error.to_string().contains(expected),
3484 "{label}: wanted {expected:?}, got {error}"
3485 );
3486 assert_eq!(
3487 recorder.calls().len(),
3488 0,
3489 "{label}: the refusal has to be local, because a launch that reached AWS costs a \
3490 VM and a bill"
3491 );
3492 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3493 assert!(sandbox.microvm().is_none());
3494 }
3495 }
3496
3497 #[tokio::test]
3504 async fn a_launch_that_dies_during_startup_installs_no_token() {
3505 let (mut sandbox, recorder, _) = planted();
3506 recorder
3507 .answer(
3508 "RunMicrovm",
3509 Answer::ok(fake::microvm_response("PENDING", None)),
3510 )
3511 .answer(
3512 "GetMicrovm",
3513 Answer::ok(fake::microvm_response(
3514 "TERMINATED",
3515 Some("run hook returned 500"),
3516 )),
3517 );
3518
3519 let error = sandbox
3520 .run(RunRequest::new().with_image("arn:image"))
3521 .await
3522 .expect_err("a VM that died during startup is not a launch");
3523 assert_eq!(error.kind(), ErrorKind::LaunchDied);
3524 assert!(
3525 error.to_string().contains("run hook returned 500"),
3526 "{error}"
3527 );
3528
3529 assert!(
3530 !sandbox.token_installed(),
3531 "the run hook never answered, so nothing was installed"
3532 );
3533 assert_eq!(
3534 sandbox.bootstrap_count(),
3535 0,
3536 "a bootstrap counted here would make STATE-3's ceiling unreachable for a retry"
3537 );
3538 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
3539 assert!(
3540 sandbox.image_exists(),
3541 "STATE-1 still holds: the launch was accepted against a real image"
3542 );
3543 }
3544
3545 #[tokio::test]
3556 async fn a_second_run_on_one_sandbox_is_refused_before_any_call() {
3557 let (mut sandbox, recorder, _) = launched().await;
3558 assert_eq!(sandbox.bootstrap_count(), 1);
3559 let before = recorder.calls().len();
3560
3561 let error = sandbox
3562 .run(RunRequest::new().with_image("arn:image"))
3563 .await
3564 .expect_err("the token is installed at most once per VM lifetime");
3565 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3566 assert!(error.to_string().contains("STATE-3"), "{error}");
3567 assert!(
3568 error.to_string().contains("needs a second Sandbox"),
3569 "the remedy has to be nameable: {error}"
3570 );
3571
3572 assert_eq!(
3573 sandbox.bootstrap_count(),
3574 1,
3575 "bootstrap_count <= 1 is the Z3-proved invariant"
3576 );
3577 assert_eq!(
3578 recorder.calls().len(),
3579 before,
3580 "the refusal must cost no control-plane call"
3581 );
3582 assert_eq!(recorder.call_count("RunMicrovm"), 1);
3583 }
3584
3585 #[tokio::test]
3588 async fn a_suspend_from_running_passes_through_suspending_to_suspended() {
3589 let (sandbox, recorder, _) = suspended().await;
3590 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3591 assert!(
3592 sandbox.token_installed(),
3593 "a freeze keeps the guest's memory, so the token survives"
3594 );
3595 assert_eq!(sandbox.bootstrap_count(), 1, "no re-bootstrap on a freeze");
3596
3597 let calls = recorder.calls();
3598 let suspend = calls
3599 .iter()
3600 .find(|call| call.operation == "SuspendMicrovm")
3601 .expect("the suspend went out");
3602 assert_eq!(suspend.method, Method::Post);
3603 assert_eq!(suspend.path, "/2025-09-09/microvms/mvm-abc123/suspend");
3604 }
3605
3606 #[tokio::test]
3612 async fn a_vm_that_dies_while_suspending_is_recorded_rather_than_raised() {
3613 let (mut sandbox, recorder, _) = launched().await;
3614 recorder
3615 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
3616 .answer(
3617 "GetMicrovm",
3618 Answer::ok(fake::microvm_response("TERMINATED", Some("idle policy"))),
3619 );
3620
3621 sandbox
3622 .suspend()
3623 .await
3624 .expect("a death mid-suspend is a state, not an error");
3625 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
3626 assert!(
3627 sandbox.was_terminated(),
3628 "STATE-11's precondition is recorded"
3629 );
3630 }
3631
3632 #[tokio::test]
3641 async fn a_suspend_whose_call_fails_stays_running_and_can_be_retried() {
3642 let (mut sandbox, recorder, _) = planted();
3643 recorder
3644 .answer(
3645 "RunMicrovm",
3646 Answer::ok(fake::microvm_response("PENDING", None)),
3647 )
3648 .answer(
3649 "GetMicrovm",
3650 Answer::ok(fake::microvm_response("RUNNING", None)),
3651 )
3652 .answer(
3653 "GetMicrovm",
3654 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3655 )
3656 .answer(
3657 "CreateMicrovmAuthToken",
3658 Answer::ok(fake::auth_token_response("proxy-token")),
3659 )
3660 .answer("SuspendMicrovm", Answer::failure(409, "ConflictException"))
3663 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3664
3665 sandbox
3666 .run(RunRequest::new().with_image("arn:image"))
3667 .await
3668 .expect("launches");
3669
3670 sandbox
3671 .suspend()
3672 .await
3673 .expect_err("the control plane refused the suspend");
3674 assert_eq!(
3675 sandbox.lifecycle(),
3676 Lifecycle::Running,
3677 "a refused suspend must not move the lifecycle: SUSPENDING accepts neither a \
3678 suspend nor a resume, so recording it here would brick the handle"
3679 );
3680
3681 sandbox.suspend().await.expect("the retry suspends");
3682 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended);
3683 assert_eq!(recorder.call_count("SuspendMicrovm"), 2);
3684 }
3685
3686 #[tokio::test]
3698 async fn a_suspend_from_a_non_running_state_reaches_no_control_plane_call() {
3699 let (mut sandbox, recorder, _) = suspended().await;
3701 let before = recorder.call_count("SuspendMicrovm");
3702
3703 let error = sandbox
3704 .suspend()
3705 .await
3706 .expect_err("a suspend is only issued from RUNNING");
3707 assert_eq!(error.kind(), ErrorKind::InvalidArg);
3708 assert!(error.to_string().contains("STATE-5"), "{error}");
3709 assert!(error.to_string().contains("SUSPENDED"), "{error}");
3710 assert_eq!(
3711 recorder.call_count("SuspendMicrovm"),
3712 before,
3713 "the refusal must be local: no second suspend went to the wire"
3714 );
3715
3716 let (mut fresh, fresh_recorder, _) = planted();
3718 let error = fresh.suspend().await.expect_err("nothing to suspend");
3719 assert_eq!(error.kind(), ErrorKind::Precondition);
3720 assert_eq!(fresh_recorder.calls().len(), 0);
3721 }
3722
3723 #[tokio::test]
3734 async fn a_resume_reuses_the_token_redelivers_nothing_and_drops_the_proxy_token() {
3735 let (mut sandbox, recorder, _) = suspended().await;
3736
3737 let auth = Arc::clone(
3739 sandbox
3740 .session()
3741 .expect("launched")
3742 .proxy_auth()
3743 .expect("the launch wired a minter"),
3744 );
3745 auth.headers().await.expect("mints");
3746 assert!(
3747 auth.is_cached(),
3748 "there must be a token for the drop to matter"
3749 );
3750 assert_eq!(auth.mint_count(), 1);
3751
3752 recorder
3753 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3754 .answer(
3755 "GetMicrovm",
3756 Answer::ok(fake::microvm_response("RUNNING", None)),
3757 );
3758 let session = sandbox.resume().await.expect("resumes");
3759 assert_eq!(
3760 session.endpoint(),
3761 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
3762 );
3763
3764 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3765 assert!(
3766 sandbox.token_installed(),
3767 "STATE-7: the installed token is reused"
3768 );
3769 assert_eq!(
3770 sandbox.bootstrap_count(),
3771 1,
3772 "STATE-7: a resume re-delivers no run-hook payload"
3773 );
3774 assert_eq!(
3775 recorder.call_count("RunMicrovm"),
3776 1,
3777 "a resume is not a second launch"
3778 );
3779 assert!(
3780 !auth.is_cached(),
3781 "STATE-8: the cached proxy token survived the resume"
3782 );
3783 }
3784
3785 #[tokio::test]
3795 async fn the_request_after_a_resume_mints_a_fresh_proxy_token() {
3796 let (mut sandbox, recorder, _) = suspended().await;
3797 let auth = Arc::clone(
3798 sandbox
3799 .session()
3800 .expect("launched")
3801 .proxy_auth()
3802 .expect("wired"),
3803 );
3804 auth.headers().await.expect("mints");
3805 assert_eq!(auth.mint_count(), 1);
3806
3807 recorder
3808 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3809 .answer(
3810 "GetMicrovm",
3811 Answer::ok(fake::microvm_response("RUNNING", None)),
3812 );
3813 sandbox.resume().await.expect("resumes");
3814
3815 auth.headers().await.expect("re-mints");
3818 assert_eq!(
3819 auth.mint_count(),
3820 2,
3821 "STATE-8: the resume did not force a fresh mint"
3822 );
3823 }
3824
3825 #[tokio::test]
3844 async fn the_lifecycle_watch_follows_every_transition() {
3845 let (mut sandbox, recorder, _clock) = planted();
3846 let watch = sandbox.watch_lifecycle();
3847 assert_eq!(*watch.borrow(), Lifecycle::Pending);
3848 recorder
3849 .answer(
3850 "RunMicrovm",
3851 Answer::ok(fake::microvm_response("PENDING", None)),
3852 )
3853 .answer(
3854 "GetMicrovm",
3855 Answer::ok(fake::microvm_response("RUNNING", None)),
3856 )
3857 .answer(
3858 "GetMicrovm",
3859 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3860 )
3861 .answer(
3862 "CreateMicrovmAuthToken",
3863 Answer::ok(fake::auth_token_response("proxy-token")),
3864 )
3865 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()));
3866 sandbox
3867 .run(RunRequest::new().with_image("arn:image"))
3868 .await
3869 .expect("launches");
3870 assert_eq!(*watch.borrow(), Lifecycle::Running);
3871 sandbox.suspend().await.expect("suspends");
3872 assert_eq!(*watch.borrow(), Lifecycle::Suspended);
3873 assert_eq!(*watch.borrow(), sandbox.lifecycle());
3874 }
3875
3876 #[tokio::test]
3877 async fn a_resume_past_the_suspended_window_is_refused_before_the_wire() {
3878 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
3880 assert_eq!(sandbox.suspended_window(), Some(Duration::from_secs(60)));
3881 assert_eq!(
3882 sandbox.idle_window(),
3883 Some(Duration::from_secs(600)),
3884 "the launch's own maxIdleDurationSeconds, which a keepalive checks its cadence against"
3885 );
3886
3887 clock.advance(Duration::from_secs(61));
3889
3890 recorder
3893 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3894 .answer(
3895 "GetMicrovm",
3896 Answer::ok(fake::microvm_response(
3897 "TERMINATED",
3898 Some("suspended window elapsed"),
3899 )),
3900 );
3901 let elapsed_before = clock.now();
3902 let resumes_before = recorder.call_count("ResumeMicrovm");
3903
3904 let error = sandbox
3905 .resume()
3906 .await
3907 .expect_err("the window the launch set has closed");
3908
3909 assert_eq!(error.kind(), ErrorKind::WindowClosed);
3910 assert_eq!(error.code(), "ERR_WINDOW_CLOSED");
3911 let message = error.to_string();
3912 assert!(message.contains("61s"), "the elapsed time: {message}");
3913 assert!(
3914 message.contains("60s suspendedDurationSeconds"),
3915 "the window has to be named: {message}"
3916 );
3917 assert!(message.contains("idlePolicy"), "the finding: {message}");
3918 assert!(
3919 message.contains("Refused before ResumeMicrovm"),
3920 "why the refusal is local: {message}"
3921 );
3922 assert_eq!(
3923 recorder.call_count("ResumeMicrovm"),
3924 resumes_before,
3925 "the refusal must come before ResumeMicrovm"
3926 );
3927 assert_eq!(
3928 clock.now(),
3929 elapsed_before,
3930 "the refusal must be immediate rather than polled to the deadline"
3931 );
3932 }
3933
3934 #[tokio::test]
3937 async fn the_window_falls_back_to_the_idle_policy_the_service_reported() {
3938 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
3939 sandbox.suspended_window = None;
3940 recorder.answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
3941 clock.advance(Duration::from_secs(601));
3942 let error = sandbox
3943 .resume()
3944 .await
3945 .expect_err("past the reported window");
3946 assert_eq!(error.kind(), ErrorKind::WindowClosed);
3947 assert!(error.to_string().contains("600s"), "{error}");
3948 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
3949 }
3950
3951 #[tokio::test]
3955 async fn a_client_token_launch_resumes_the_vm_it_adopted() {
3956 let (mut sandbox, recorder, _) = planted();
3957 recorder
3958 .answer(
3959 "RunMicrovm",
3960 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3961 )
3962 .answer(
3963 "GetMicrovm",
3964 Answer::ok(fake::microvm_response("SUSPENDED", None)),
3965 )
3966 .answer(
3967 "GetMicrovm",
3968 Answer::ok(fake::microvm_response("RUNNING", None)),
3969 )
3970 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
3971 .answer(
3972 "CreateMicrovmAuthToken",
3973 Answer::ok(fake::auth_token_response("proxy-token")),
3974 );
3975 let mut request = RunRequest::new().with_image("arn:image");
3976 request.client_token = Some("job-195".into());
3977 request.agent_token = Some("agent-token".into());
3978 sandbox
3979 .run(request)
3980 .await
3981 .expect("the adopted VM is resumed");
3982 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
3983 assert_eq!(sandbox.bootstrap_count(), 1);
3984 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
3985
3986 let (mut fresh, recorder, _) = planted();
3987 recorder
3988 .answer(
3989 "RunMicrovm",
3990 Answer::ok(fake::microvm_response("PENDING", None)),
3991 )
3992 .answer(
3993 "GetMicrovm",
3994 Answer::ok(fake::microvm_response("SUSPENDED", Some("hook failed"))),
3995 );
3996 let error = fresh
3997 .run(RunRequest::new().with_image("arn:image"))
3998 .await
3999 .expect_err("a fresh launch that reads SUSPENDED died during startup");
4000 assert_eq!(error.kind(), ErrorKind::LaunchDied);
4001 assert_eq!(recorder.call_count("ResumeMicrovm"), 0);
4002 }
4003
4004 #[tokio::test]
4007 async fn a_launch_that_does_not_wait_is_finished_by_wait_until_running() {
4008 let (mut sandbox, recorder, _) = planted();
4009 answer_launch(&recorder);
4010 let mut request = RunRequest::new().with_image("arn:image");
4011 request.wait = false;
4012 let endpoint = sandbox
4013 .run(request)
4014 .await
4015 .expect("accepted")
4016 .endpoint()
4017 .to_string();
4018 assert!(endpoint.starts_with("https://"), "{endpoint}");
4019 assert_eq!(sandbox.lifecycle(), Lifecycle::Pending);
4020 assert_eq!(sandbox.bootstrap_count(), 0);
4021 assert_eq!(recorder.call_count("GetMicrovm"), 0, "nothing polled yet");
4022
4023 sandbox
4024 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4025 .await
4026 .expect("reaches RUNNING");
4027 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4028 assert_eq!(sandbox.bootstrap_count(), 1);
4029 let error = sandbox
4030 .wait_until_running(DEFAULT_RUNNING_TIMEOUT)
4031 .await
4032 .expect_err("a running launch has nothing to wait for");
4033 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4034 assert_eq!(sandbox.bootstrap_count(), 1, "counted once (STATE-3)");
4035 }
4036
4037 #[tokio::test]
4043 async fn a_resume_inside_the_window_is_allowed_and_the_boundary_is_inclusive() {
4044 let (mut sandbox, recorder, clock) = suspended_with_window(60).await;
4045
4046 clock.advance(Duration::from_secs(60));
4048 recorder
4049 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4050 .answer(
4051 "GetMicrovm",
4052 Answer::ok(fake::microvm_response("RUNNING", None)),
4053 );
4054 sandbox
4055 .resume()
4056 .await
4057 .expect("60s == the 60s window is open");
4058 assert_eq!(sandbox.lifecycle(), Lifecycle::Running);
4059 assert_eq!(recorder.call_count("ResumeMicrovm"), 1);
4060 }
4061
4062 #[tokio::test]
4070 async fn a_successful_resume_clears_the_stamp_so_a_second_cycle_measures_its_own_window() {
4071 let (mut sandbox, recorder, clock) = planted();
4072 recorder
4077 .answer(
4078 "RunMicrovm",
4079 Answer::ok(fake::microvm_response("PENDING", None)),
4080 )
4081 .answer(
4082 "CreateMicrovmAuthToken",
4083 Answer::ok(fake::auth_token_response("proxy-token")),
4084 )
4085 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
4086 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
4087 .answer(
4088 "GetMicrovm",
4089 Answer::ok(fake::microvm_response("RUNNING", None)),
4090 )
4091 .answer(
4092 "GetMicrovm",
4093 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4094 )
4095 .answer(
4096 "GetMicrovm",
4097 Answer::ok(fake::microvm_response("RUNNING", None)),
4098 )
4099 .answer(
4100 "GetMicrovm",
4101 Answer::ok(fake::microvm_response("SUSPENDED", None)),
4102 )
4103 .answer(
4104 "GetMicrovm",
4105 Answer::ok(fake::microvm_response("RUNNING", None)),
4106 );
4107 sandbox
4108 .run(
4109 RunRequest::new()
4110 .with_image("arn:image")
4111 .with_suspended_sec(60),
4112 )
4113 .await
4114 .expect("launches");
4115
4116 for cycle in 0..2 {
4117 sandbox
4118 .suspend()
4119 .await
4120 .unwrap_or_else(|error| panic!("cycle {cycle} suspend: {error}"));
4121 assert_eq!(sandbox.lifecycle(), Lifecycle::Suspended, "cycle {cycle}");
4122
4123 clock.advance(Duration::from_secs(45));
4127
4128 sandbox.resume().await.unwrap_or_else(|error| {
4129 panic!("cycle {cycle} resume, whose own window is wide open: {error}")
4130 });
4131 assert_eq!(sandbox.lifecycle(), Lifecycle::Running, "cycle {cycle}");
4132 assert_eq!(
4133 sandbox.bootstrap_count(),
4134 1,
4135 "cycle {cycle}: no re-bootstrap"
4136 );
4137 }
4138
4139 assert_eq!(
4140 clock.now(),
4141 Duration::from_secs(90),
4142 "ninety seconds of accumulated suspension against a sixty-second window: this is \
4143 the total an implementation that never cleared the stamp would have compared"
4144 );
4145 assert_eq!(
4146 recorder.call_count("ResumeMicrovm"),
4147 2,
4148 "both cycles resumed"
4149 );
4150 }
4151
4152 #[tokio::test]
4160 async fn a_resume_after_terminate_records_zero_resume_calls() {
4161 let (mut sandbox, recorder, _) = launched().await;
4162 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4163 let report = sandbox.terminate(TeardownOpts::default()).await;
4164 assert!(report.terminate_accepted);
4165 assert!(sandbox.was_terminated());
4166
4167 let error = sandbox
4168 .resume()
4169 .await
4170 .expect_err("a terminated VM never returns to RUNNING");
4171 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4172 assert!(error.to_string().contains("STATE-11"), "{error}");
4173 assert_eq!(
4174 recorder.call_count("ResumeMicrovm"),
4175 0,
4176 "no resume may reach the wire once the VM was terminated"
4177 );
4178 assert_ne!(
4179 sandbox.lifecycle(),
4180 Lifecycle::Running,
4181 "the lifecycle must not return to RUNNING"
4182 );
4183 }
4184
4185 #[tokio::test]
4192 async fn a_terminate_records_terminating_and_the_completion_report_marks_terminated() {
4193 let (mut sandbox, recorder, _) = launched().await;
4194 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4195
4196 let report = sandbox.terminate(TeardownOpts::default()).await;
4197 assert!(report.terminate_accepted, "STATE-9: the call was accepted");
4198 assert!(sandbox.was_terminated(), "STATE-9: recorded terminated");
4199 assert_eq!(
4200 report.lifecycle,
4201 Some(Lifecycle::Terminating),
4202 "the default teardown does not wait, so TERMINATED is not claimed"
4203 );
4204 assert!(!report.leaked(), "{report:?}");
4205
4206 let (mut sandbox, recorder, _) = launched().await;
4208 recorder
4209 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4210 .answer(
4211 "GetMicrovm",
4212 Answer::ok(fake::microvm_response("TERMINATED", None)),
4213 );
4214 let report = sandbox
4215 .terminate(TeardownOpts::default().waiting_for_terminated())
4216 .await;
4217 assert_eq!(report.lifecycle, Some(Lifecycle::Terminated));
4218 assert_eq!(sandbox.lifecycle(), Lifecycle::Terminated);
4219 }
4220
4221 #[tokio::test]
4230 async fn teardown_deletes_the_vm_then_the_image_and_names_the_log_group_last() {
4231 let (mut sandbox, recorder, _) = planted();
4232 recorder
4233 .answer(
4234 "CreateMicrovmImage",
4235 Answer::created(fake::create_image_response("agentd-conformance")),
4236 )
4237 .answer(
4238 "GetMicrovmImage",
4239 Answer::ok(fake::get_image_response("agentd-conformance", "CREATED")),
4240 );
4241 sandbox
4242 .build_image(CreateImageRequest::new(
4243 "agentd-conformance",
4244 crate::testing::aarch64_daemon(b"binary"),
4245 "s3://bucket/img.zip",
4246 "arn:aws:iam::123456789012:role/build",
4247 ))
4248 .await
4249 .expect("builds");
4250 answer_launch(&recorder);
4251 sandbox
4252 .run(RunRequest::new())
4253 .await
4254 .expect("launches from the built image");
4255
4256 recorder
4257 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4258 .answer(
4259 "ListMicrovmImageVersions",
4260 Answer::ok(fake::list_versions_response("1")),
4261 )
4262 .answer(
4263 "DeleteMicrovmImage",
4264 Answer::ok(fake::delete_image_response()),
4265 );
4266
4267 let report = sandbox
4268 .terminate(
4269 TeardownOpts::default()
4270 .deleting_image()
4271 .deleting_log_group(),
4272 )
4273 .await;
4274 assert!(report.terminate_accepted);
4275 assert_eq!(report.image_deleted, Some(true));
4276
4277 let order: Vec<&str> = recorder
4279 .operations()
4280 .into_iter()
4281 .filter(|operation| matches!(*operation, "TerminateMicrovm" | "DeleteMicrovmImage"))
4282 .collect();
4283 assert_eq!(
4284 order,
4285 ["TerminateMicrovm", "DeleteMicrovmImage"],
4286 "the VM goes before the image: a terminating VM holds a reference to it"
4287 );
4288
4289 assert_eq!(
4291 report.undeleted,
4292 ["/aws/lambda-microvms/agentd-conformance"],
4293 "{report:?}"
4294 );
4295 assert!(report.leaked());
4296 assert!(
4297 report
4298 .failures
4299 .last()
4300 .expect("a failure")
4301 .contains("log group"),
4302 "the log group's failure has to be the last one recorded: {report:?}"
4303 );
4304 assert!(
4305 report
4306 .failures
4307 .last()
4308 .expect("a failure")
4309 .contains("terraform destroy"),
4310 "the reason it leaks has to be named: {report:?}"
4311 );
4312 }
4313
4314 #[tokio::test]
4320 async fn a_failing_image_delete_names_what_it_left_behind() {
4321 let (mut sandbox, recorder, _) = planted();
4322 recorder
4323 .answer(
4324 "CreateMicrovmImage",
4325 Answer::created(fake::create_image_response("img")),
4326 )
4327 .answer(
4328 "GetMicrovmImage",
4329 Answer::ok(fake::get_image_response("img", "CREATED")),
4330 );
4331 sandbox
4332 .build_image(CreateImageRequest::new(
4333 "img",
4334 crate::testing::aarch64_daemon(b"binary"),
4335 "s3://bucket/img.zip",
4336 "arn:aws:iam::123456789012:role/build",
4337 ))
4338 .await
4339 .expect("builds");
4340 answer_launch(&recorder);
4341 sandbox.run(RunRequest::new()).await.expect("launches");
4342
4343 recorder
4344 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
4345 .answer(
4346 "ListMicrovmImageVersions",
4347 Answer::ok(fake::list_versions_response("1")),
4348 )
4349 .answer(
4350 "DeleteMicrovmImage",
4351 Answer::failure(409, "image is in CREATING"),
4352 );
4353
4354 let mut opts = TeardownOpts::default().deleting_image();
4355 opts.delete_attempts = 3;
4356 opts.delete_backoff = Duration::from_secs(1);
4357 let report = sandbox.terminate(opts).await;
4358
4359 assert_eq!(report.image_deleted, Some(false));
4360 assert_eq!(
4361 report.undeleted,
4362 ["arn:aws:lambda:us-east-1:123456789012:microvm-image:img"],
4363 "{report:?}"
4364 );
4365 assert!(
4366 sandbox.image_exists(),
4367 "the image really is still there, so the flag must say so"
4368 );
4369 assert_eq!(
4370 recorder.call_count("DeleteMicrovmImage"),
4371 3,
4372 "every attempt ran"
4373 );
4374 }
4375
4376 #[tokio::test]
4383 async fn a_terminate_whose_call_fails_still_records_the_intent_and_the_leak() {
4384 let (mut sandbox, recorder, _) = launched().await;
4385 recorder.answer(
4386 "TerminateMicrovm",
4387 Answer::failure(500, "InternalServerException"),
4388 );
4389
4390 let report = sandbox.terminate(TeardownOpts::default()).await;
4391 assert!(!report.terminate_accepted);
4392 assert_eq!(report.undeleted, ["mvm-abc123"], "{report:?}");
4393 assert!(sandbox.was_terminated());
4394 assert_eq!(report.lifecycle, Some(Lifecycle::Terminating));
4395 assert!(
4396 !report.failures.is_empty(),
4397 "the swallowed failure is recorded"
4398 );
4399 }
4400
4401 #[tokio::test]
4407 async fn a_teardown_before_any_launch_is_an_empty_report() {
4408 let (mut sandbox, recorder, _) = planted();
4409 let report = sandbox.terminate(TeardownOpts::default()).await;
4410 assert!(!report.terminate_accepted);
4411 assert!(!report.leaked());
4412 assert_eq!(report.lifecycle, Some(Lifecycle::Pending));
4413 assert_eq!(recorder.calls().len(), 0);
4414 }
4415
4416 #[tokio::test]
4419 async fn a_run_with_no_image_is_refused_before_any_call() {
4420 let (mut sandbox, recorder, _) = planted();
4421 let error = sandbox
4422 .run(RunRequest::new())
4423 .await
4424 .expect_err("there is nothing to launch");
4425 assert_eq!(error.kind(), ErrorKind::Precondition);
4426 assert!(error.to_string().contains("build_image first"), "{error}");
4427 assert_eq!(recorder.calls().len(), 0);
4428 }
4429
4430 #[test]
4437 fn the_lifecycle_states_are_the_state_models_domain_in_the_services_spelling() {
4438 let all = [
4439 Lifecycle::Pending,
4440 Lifecycle::Running,
4441 Lifecycle::Suspending,
4442 Lifecycle::Suspended,
4443 Lifecycle::Terminating,
4444 Lifecycle::Terminated,
4445 ];
4446 assert_eq!(
4447 all.map(Lifecycle::as_str),
4448 [
4449 "PENDING",
4450 "RUNNING",
4451 "SUSPENDING",
4452 "SUSPENDED",
4453 "TERMINATING",
4454 "TERMINATED"
4455 ]
4456 );
4457 for state in all {
4458 assert_eq!(state.to_string(), state.as_str());
4459 }
4460 assert_eq!(
4463 all.iter().filter(|state| state.is_live()).count(),
4464 4,
4465 "a state added to the enum defaults to not-live, which would silence the warning"
4466 );
4467 assert!(!Lifecycle::Terminating.is_live());
4468 assert!(!Lifecycle::Terminated.is_live());
4469 }
4470
4471 #[test]
4479 fn a_minted_agent_token_is_sixty_four_hex_characters_and_never_repeats() {
4480 let entropy = crate::entropy::testing::SequenceEntropy::new();
4481 let minted: std::collections::HashSet<String> = (0..200)
4482 .map(|_| mint_agent_token(&entropy).expect("scripted"))
4483 .collect();
4484 assert_eq!(minted.len(), 200, "a repeated token is a shared guest");
4485 for token in &minted {
4486 assert_eq!(token.len(), 64, "{token}");
4487 assert!(token.chars().all(|c| c.is_ascii_hexdigit()), "{token}");
4488 assert_ne!(
4489 token,
4490 &"0".repeat(64),
4491 "an all-zero draw is a read that did nothing"
4492 );
4493 }
4494 }
4495
4496 #[tokio::test]
4498 async fn a_sandbox_debug_does_not_print_the_agent_token() {
4499 let (mut sandbox, recorder, _) = planted();
4500 answer_launch(&recorder);
4501 sandbox
4502 .run(
4503 RunRequest::new()
4504 .with_image("arn:image")
4505 .with_agent_token("super-secret-agent-token"),
4506 )
4507 .await
4508 .expect("launches");
4509
4510 let rendered = format!("{sandbox:?}");
4511 assert!(rendered.contains("mvm-abc123"), "{rendered}");
4512 assert!(rendered.contains("Running"), "{rendered}");
4513 assert!(
4514 !rendered.contains("super-secret"),
4515 "the agent token reached a Debug string: {rendered}"
4516 );
4517 }
4518
4519 #[tokio::test]
4527 async fn a_torn_down_sandbox_is_distinguishable_from_an_abandoned_one() {
4528 let (mut sandbox, recorder, _) = launched().await;
4529 assert!(
4530 sandbox.lifecycle().is_live() && !sandbox.torn_down,
4531 "an abandoned live VM is what the drop warning is for"
4532 );
4533
4534 recorder.answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
4535 sandbox.terminate(TeardownOpts::default()).await;
4536 assert!(
4537 sandbox.torn_down,
4538 "terminate must mark the sandbox torn down"
4539 );
4540 assert!(
4541 !sandbox.lifecycle().is_live(),
4542 "and the lifecycle must no longer read as live"
4543 );
4544 }
4545
4546 #[tokio::test]
4547 async fn stable_launch_key_and_payload_survive_a_fresh_supervisor() {
4548 let mut bodies = Vec::new();
4549 for _ in 0..2 {
4550 let (mut sandbox, recorder, _) = planted();
4551 answer_launch(&recorder);
4552 let mut request = RunRequest::new().with_image("arn:image");
4553 request.client_token = Some("one-unique-job-launch".into());
4554 request.agent_token = Some("persisted-private-secret".into());
4555 sandbox.run(request).await.expect("accepted");
4556 bodies.push(recorder.first_body("RunMicrovm"));
4557 }
4558 assert_eq!(bodies[0]["clientToken"], "one-unique-job-launch");
4559 assert_eq!(bodies[0], bodies[1]);
4560 }
4561
4562 #[tokio::test]
4563 async fn unsafe_or_malformed_stable_launch_is_refused_without_calls() {
4564 for (token, secret, identity) in [
4565 ("key".to_string(), None, false),
4566 ("key".to_string(), Some("secret".into()), true),
4567 ("".to_string(), Some("secret".into()), false),
4568 ("x".repeat(129), Some("secret".into()), false),
4569 ("line\nbreak".to_string(), Some("secret".into()), false),
4570 ] {
4571 let (mut sandbox, recorder, _) = planted();
4572 let mut request = RunRequest::new().with_image("arn:image");
4573 request.client_token = Some(token);
4574 request.agent_token = secret;
4575 request.identity = identity;
4576 let error = sandbox.run(request).await.expect_err("invalid");
4577 assert_eq!(error.kind(), ErrorKind::InvalidArg);
4578 assert!(recorder.calls().is_empty());
4579 }
4580 }
4581}