1use super::transport::{Call, paths, send_with_retry};
49use super::{ControlPlane, RunMicrovmRequest, WaitOpts, ops, timed_out, token};
50use crate::error::{Error, ErrorKind};
51
52pub const PROXY_AUTH_HEADER: &str = "X-aws-proxy-auth";
54
55pub const PROXY_PORT_HEADER: &str = "X-aws-proxy-port";
61
62pub const MAX_TOKEN_MINUTES: u32 = 60;
64
65#[derive(Clone, Eq, PartialEq)]
71pub struct RunHookPayload(String);
72
73impl std::fmt::Debug for RunHookPayload {
82 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
83 write!(f, "RunHookPayload(<{} bytes>)", self.0.len())
84 }
85}
86
87impl RunHookPayload {
88 pub fn for_agent_token(agent_token: &str) -> Result<Self, Error> {
97 Self::for_launch(agent_token, &std::collections::HashMap::new())
98 }
99
100 pub fn for_launch(
126 agent_token: &str,
127 env: &std::collections::HashMap<String, String>,
128 ) -> Result<Self, Error> {
129 Self::for_launch_with_identity(agent_token, env, None)
130 }
131
132 pub fn for_launch_with_identity(
145 agent_token: &str,
146 env: &std::collections::HashMap<String, String>,
147 identity: Option<&crate::identity::LaunchIdentity>,
148 ) -> Result<Self, Error> {
149 let mut payload = if env.is_empty() {
150 serde_json::json!({ "agent_token": agent_token })
151 } else {
152 serde_json::json!({ "agent_token": agent_token, "env": env })
153 };
154 if let Some(identity) = identity {
155 let object = payload
156 .as_object_mut()
157 .expect("the payload is built as an object two lines up");
158 object.insert(
159 protocol::identity::SEED_KEY.to_string(),
160 serde_json::Value::String(identity.seed_field()),
161 );
162 object.insert(
163 protocol::identity::HOST_PUBLIC_KEY_KEY.to_string(),
164 serde_json::Value::String(identity.host_public_field()),
165 );
166 }
167 let text = serde_json::to_string(&payload).map_err(|error| {
168 Error::new(
169 ErrorKind::Unexpected,
170 format!("could not serialize the run-hook payload: {error}"),
171 )
172 })?;
173 let total = text.len();
176 Self::new(text).map_err(|error| {
177 if env.is_empty() {
178 return error;
179 }
180 Error::invalid_arg(format!(
185 "{error} The launch env contributed {} of those bytes across {} variable(s); \
186 the payload's other {} are the token and the JSON framing. A launch env is \
187 for small values a workload needs at startup — move credential-scale \
188 material to PUT /v1/fs/file after bootstrap, or to a role the workload \
189 assumes.",
190 env_contribution(agent_token, env),
191 env.len(),
192 total.saturating_sub(env_contribution(agent_token, env)),
193 ))
194 })
195 }
196
197 pub fn new(payload: impl Into<String>) -> Result<Self, Error> {
202 let payload = payload.into();
203 let size = payload.len();
207 if size <= crate::constants::MAX_RUN_HOOK_PAYLOAD_BYTES {
208 return Ok(Self(payload));
209 }
210 Err(Error::invalid_arg(format!(
211 "runHookPayload is {size} bytes, over the ceiling of {} (service model {}, measured \
212 inclusive 2026-08-07: 4096 passes and 4097 fails). This is the only per-VM secret \
213 channel the platform offers — one bearer token fits, a cloud credential set does \
214 not. Note that docs/STRATEGY.md, docs/TRUST.md, and the service model's own \
215 documentation string for this member all claim 16 KB, which is wrong by 4x in the \
216 dangerous direction; the shape RunMicrovmRequestRunHookPayloadString is the \
217 authority and it states max 4096.",
218 crate::constants::MAX_RUN_HOOK_PAYLOAD_BYTES,
219 crate::constants::MODEL_API_VERSION,
220 )))
221 }
222
223 pub fn as_str(&self) -> &str {
225 &self.0
226 }
227
228 pub fn len(&self) -> usize {
230 self.0.len()
231 }
232
233 pub fn is_empty(&self) -> bool {
235 self.0.is_empty()
236 }
237}
238
239fn env_contribution(agent_token: &str, env: &std::collections::HashMap<String, String>) -> usize {
249 let with = serde_json::json!({ "agent_token": agent_token, "env": env }).to_string();
250 let without = serde_json::json!({ "agent_token": agent_token }).to_string();
251 with.len().saturating_sub(without.len())
252}
253
254#[derive(Clone, Debug, Eq, PartialEq)]
256pub struct Microvm {
257 pub id: String,
258 pub state: String,
259 pub endpoint: String,
261 pub image_arn: String,
262 pub image_version: String,
263 pub state_reason: Option<String>,
266 pub idle_policy: Option<ops::IdlePolicy>,
277 pub maximum_duration_seconds: Option<u32>,
279 pub started_at: Option<std::time::SystemTime>,
281 pub terminated_at: Option<std::time::SystemTime>,
283}
284
285fn epoch(seconds: Option<f64>) -> Option<std::time::SystemTime> {
291 let since = std::time::Duration::try_from_secs_f64(seconds?).ok()?;
292 std::time::UNIX_EPOCH.checked_add(since)
293}
294
295#[derive(Clone, Debug, Default, Eq, PartialEq)]
297pub struct MicrovmFilter {
298 pub image_identifier: Option<String>,
300 pub image_version: Option<String>,
302}
303
304impl Microvm {
305 pub(crate) fn reason(&self) -> &str {
311 self.state_reason.as_deref().unwrap_or("no stateReason")
312 }
313}
314
315impl From<ops::MicrovmResponseWire> for Microvm {
316 fn from(wire: ops::MicrovmResponseWire) -> Self {
317 Self {
318 id: wire.microvm_id,
319 state: wire.state,
320 endpoint: wire.endpoint,
321 image_arn: wire.image_arn,
322 image_version: wire.image_version,
323 state_reason: wire.state_reason,
324 idle_policy: wire.idle_policy,
325 maximum_duration_seconds: wire.maximum_duration_in_seconds,
326 started_at: epoch(wire.started_at),
327 terminated_at: epoch(wire.terminated_at),
328 }
329 }
330}
331
332#[derive(Clone, Eq, PartialEq)]
344pub struct ProxyToken {
345 value: String,
346 port: u16,
347}
348
349impl std::fmt::Debug for ProxyToken {
356 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
357 f.debug_struct("ProxyToken")
358 .field("headers", &[PROXY_AUTH_HEADER, PROXY_PORT_HEADER])
359 .field("port", &self.port)
360 .finish()
361 }
362}
363
364impl ProxyToken {
365 pub fn from_map(
372 map: &std::collections::BTreeMap<String, String>,
373 port: u16,
374 ) -> Result<Self, Error> {
375 Self::from_operation_map("CreateMicrovmAuthToken", map, port)
376 }
377
378 pub fn from_operation_map(
384 operation: &str,
385 map: &std::collections::BTreeMap<String, String>,
386 port: u16,
387 ) -> Result<Self, Error> {
388 let value = map.get(PROXY_AUTH_HEADER).ok_or_else(|| {
389 Error::wire(
390 crate::error::WireKind::AuthTokenMint,
391 format!(
392 "the {operation} response carried no {PROXY_AUTH_HEADER} key. The \
393 authToken member is a map, not a string, and that key is the header value \
394 (keys present: {:?}).",
395 map.keys().collect::<Vec<_>>()
396 ),
397 )
398 })?;
399 Ok(Self {
400 value: value.clone(),
401 port,
402 })
403 }
404
405 pub fn headers(&self) -> [(&'static str, String); 2] {
410 [
411 (PROXY_AUTH_HEADER, self.value.clone()),
412 (PROXY_PORT_HEADER, self.port.to_string()),
413 ]
414 }
415
416 pub fn value(&self) -> &str {
418 &self.value
419 }
420
421 pub fn port(&self) -> u16 {
423 self.port
424 }
425}
426
427impl ControlPlane {
428 pub async fn run_microvm(&self, request: RunMicrovmRequest) -> Result<Microvm, Error> {
434 let (ingress, egress) = self.launch_connectors(&request)?;
435
436 let wire = ops::RunMicrovmWire {
437 image_identifier: request.image_identifier.clone(),
438 image_version: request.image_version.clone(),
439 execution_role_arn: request.execution_role_arn.clone(),
440 ingress_network_connectors: ingress,
441 egress_network_connectors: (!egress.is_empty()).then_some(egress),
446 idle_policy: ops::IdlePolicy {
447 max_idle_duration_seconds: request.max_idle_sec,
448 suspended_duration_seconds: request.suspended_sec,
449 auto_resume_enabled: request.auto_resume,
450 },
451 maximum_duration_in_seconds: request.max_duration_sec,
452 run_hook_payload: request.run_hook_payload.as_str().to_string(),
453 logging: request.logging.clone(),
454 client_token: match request.client_token {
455 Some(token) => token,
456 None => token::run_token_with(
457 request
458 .token_scope
459 .as_deref()
460 .unwrap_or(&request.image_identifier),
461 self.entropy(),
462 )?,
463 },
464 };
465
466 let call = Call::post_json("RunMicrovm", paths::microvms(), &wire)?;
467 let reply = send_with_retry(self.transport(), call).await?;
468 let launched: ops::MicrovmResponseWire = reply.json("RunMicrovm")?;
469 Ok(launched.into())
470 }
471
472 pub(crate) fn launch_connectors(
478 &self,
479 request: &RunMicrovmRequest,
480 ) -> Result<(Vec<String>, Vec<String>), Error> {
481 if let Some(token) = request.client_token.as_deref() {
482 super::token::require_run_token(token)?;
483 }
484 super::require_valid_identifier("imageIdentifier", &request.image_identifier)?;
485 super::require_duration_in_range(request.max_duration_sec)?;
486 super::require_idle_duration(request.max_idle_sec)?;
490 if let Some(version) = request.image_version.as_deref() {
494 super::require_valid_version("imageVersion", version)?;
495 }
496 if let Some(role) = request.execution_role_arn.as_deref() {
500 super::require_valid_role_arn("executionRoleArn", role)?;
501 }
502 if let Some(ops::Logging::CloudWatch { cloud_watch }) = &request.logging {
503 let Some(group) = cloud_watch.log_group.as_deref() else {
504 return Err(Error::invalid_arg(
505 "per-VM CloudWatch logging needs a logGroup; pass a group name or disable logging",
506 ));
507 };
508 super::require_valid_log_group(group)?;
509 if let Some(stream) = cloud_watch.log_stream.as_deref() {
510 super::require_valid_log_stream(stream)?;
511 }
512 }
513
514 let has_all_ingress = request
521 .connectors
522 .contains(&super::ConnectorIntent::AllIngress);
523 let has_finer_ingress = request.connectors.iter().any(|intent| {
524 matches!(
525 intent,
526 super::ConnectorIntent::HttpIngress | super::ConnectorIntent::ShellIngress
527 )
528 });
529 if has_all_ingress && has_finer_ingress {
530 return Err(Error::invalid_arg(
531 "ALL_INGRESS cannot be combined with other ingress connectors; use \
532 HTTP_INGRESS and/or SHELL_INGRESS instead. The platform accepts the \
533 combination at launch and rejects it only when a shell token is minted, \
534 after the VM has launched and billed — so it is refused here.",
535 ));
536 }
537
538 let has_shell_ingress = request
543 .connectors
544 .contains(&super::ConnectorIntent::ShellIngress);
545 let has_http_ingress = request
546 .connectors
547 .contains(&super::ConnectorIntent::HttpIngress);
548 if has_shell_ingress && !has_http_ingress {
549 return Err(Error::invalid_arg(
550 "SHELL_INGRESS was requested without HTTP_INGRESS. The measured pair that \
551 launches and mints a shell token is [HTTP_INGRESS, SHELL_INGRESS] \
552 (docs/PLATFORM.md); the platform accepts an invalid ingress set at launch \
553 and the VM runs and bills before any failure appears — so it is refused \
554 here.",
555 ));
556 }
557
558 let ingress: Vec<String> = request
561 .connectors
562 .iter()
563 .filter(|intent| {
564 matches!(
565 intent,
566 super::ConnectorIntent::AllIngress
567 | super::ConnectorIntent::HttpIngress
568 | super::ConnectorIntent::ShellIngress
569 )
570 })
571 .map(|intent| intent.arn(&self.region))
572 .collect();
573 let mut egress: Vec<String> = request
574 .connectors
575 .iter()
576 .filter(|intent| matches!(intent, super::ConnectorIntent::Egress))
577 .map(|intent| intent.arn(&self.region))
578 .collect();
579
580 if !egress.is_empty() && !request.egress_network_connectors.is_empty() {
583 return Err(super::connector::egress_with_connectors());
584 }
585 for arn in &request.egress_network_connectors {
586 super::connector::require_egress_connector_arn(arn, &self.region)?;
587 egress.push(arn.clone());
588 }
589
590 for (member, count) in [
592 ("ingressNetworkConnectors", ingress.len()),
593 ("egressNetworkConnectors", egress.len()),
594 ] {
595 if let Some(refusal) = super::connector::over_connector_ceiling(member, count) {
596 return Err(refusal);
597 }
598 }
599
600 Ok((ingress, egress))
601 }
602
603 pub async fn wait_for_running(&self, id: &str, opts: WaitOpts) -> Result<Microvm, Error> {
610 self.wait_for_state(id, &["RUNNING"], &crate::constants::TERMINAL_STATES, opts)
611 .await
612 }
613
614 pub async fn wait_for_launch(&self, id: &str, opts: WaitOpts) -> Result<Microvm, Error> {
623 self.wait_for_launch_inner(id, opts, true, false).await
624 }
625
626 pub(crate) async fn wait_for_sandbox_launch(
629 &self,
630 id: &str,
631 opts: WaitOpts,
632 adoptable: bool,
633 ) -> Result<Microvm, Error> {
634 self.wait_for_launch_inner(id, opts, adoptable, true).await
635 }
636
637 async fn wait_for_launch_inner(
638 &self,
639 id: &str,
640 opts: WaitOpts,
641 adoptable: bool,
642 fast_start: bool,
643 ) -> Result<Microvm, Error> {
644 let started = self.clock().elapsed();
645 let mut resumed = false;
646 let mut fast_polls_remaining = 3;
647 let fail_on = if adoptable {
648 &crate::constants::DEAD_STATES[..]
649 } else {
650 &crate::constants::TERMINAL_STATES[..]
651 };
652 loop {
653 let got = self.get_microvm(id).await?;
654 match got.state.as_str() {
655 "RUNNING" => return Ok(got),
656 state if fail_on.contains(&state) => {
657 return Err(self.reached_terminal_state(&got, &["RUNNING"]));
658 }
659 "SUSPENDED" if adoptable && !resumed => {
662 self.resume(id).await?;
663 resumed = true;
664 continue;
665 }
666 _ => {}
667 }
668
669 let elapsed = self.clock().elapsed().saturating_sub(started);
670 if elapsed >= opts.timeout {
671 return Err(timed_out(
672 &format!(
673 "microvm {id} never reached [\"RUNNING\"] (last state {})",
674 got.state
675 ),
676 elapsed,
677 ));
678 }
679 let interval = if fast_start && fast_polls_remaining > 0 {
682 fast_polls_remaining -= 1;
683 opts.poll_interval.min(std::time::Duration::from_secs(1))
684 } else {
685 opts.poll_interval
686 };
687 self.clock().sleep(interval).await;
688 }
689 }
690
691 pub async fn wait_for_state(
702 &self,
703 id: &str,
704 wanted: &[&str],
705 fail_on: &[&str],
706 opts: WaitOpts,
707 ) -> Result<Microvm, Error> {
708 let started = self.clock().elapsed();
709 loop {
710 let got = self.get_microvm(id).await?;
711
712 if wanted.contains(&got.state.as_str()) {
713 return Ok(got);
714 }
715 if fail_on.contains(&got.state.as_str()) {
716 return Err(self.reached_terminal_state(&got, wanted));
717 }
718
719 let elapsed = self.clock().elapsed().saturating_sub(started);
720 if elapsed >= opts.timeout {
721 return Err(timed_out(
722 &format!(
723 "microvm {id} never reached {wanted:?} (last state {})",
724 got.state
725 ),
726 elapsed,
727 ));
728 }
729 self.clock().sleep(opts.poll_interval).await;
730 }
731 }
732
733 fn reached_terminal_state(&self, got: &Microvm, wanted: &[&str]) -> Error {
738 Error::new(
739 ErrorKind::LaunchDied,
740 format!(
741 "microvm {} reached {} before {wanted:?}: {}. A VM that reaches a terminal state \
742 before RUNNING died during startup, and for a hook-serving daemon that almost \
743 always means a lifecycle hook failed — the stateReason above is the only evidence \
744 left, because the VM is gone (docs/PLATFORM.md, 'MicroVM states, and terminal \
745 states reached before RUNNING').",
746 got.id,
747 got.state,
748 got.reason(),
749 ),
750 )
751 }
752
753 pub async fn get_microvm(&self, id: &str) -> Result<Microvm, Error> {
760 super::require_valid_identifier("microvmIdentifier", id)?;
761 let call = Call::get("GetMicrovm", paths::microvm(id));
762 let reply = send_with_retry(self.transport(), call).await?;
763 let got: ops::MicrovmResponseWire = reply.json("GetMicrovm")?;
764 Ok(got.into())
765 }
766
767 pub async fn list_microvms(&self) -> Result<Vec<ops::MicrovmItemWire>, Error> {
780 self.list_microvms_matching(&MicrovmFilter::default()).await
781 }
782
783 pub async fn list_microvms_matching(
785 &self,
786 filter: &MicrovmFilter,
787 ) -> Result<Vec<ops::MicrovmItemWire>, Error> {
788 if let Some(image) = filter.image_identifier.as_deref() {
789 super::require_valid_identifier("imageIdentifier", image)?;
790 }
791 if let Some(version) = filter.image_version.as_deref() {
792 super::require_valid_version("imageVersion", version)?;
793 }
794 let mut items: Vec<ops::MicrovmItemWire> = Vec::new();
795 let mut position: std::collections::HashMap<String, usize> =
799 std::collections::HashMap::new();
800 let mut next_token: Option<String> = None;
801 loop {
802 let call = Call::get(
803 "ListMicrovms",
804 paths::microvms_list_matching(
805 filter.image_identifier.as_deref(),
806 filter.image_version.as_deref(),
807 next_token.as_deref(),
808 ),
809 );
810 let reply = send_with_retry(self.transport(), call).await?;
811 let page: ops::ListMicrovmsResponseWire = reply.json("ListMicrovms")?;
812 for item in page.items {
813 match position.get(&item.microvm_id) {
814 Some(&at) => items[at] = item,
815 None => {
816 position.insert(item.microvm_id.clone(), items.len());
817 items.push(item);
818 }
819 }
820 }
821 match page.next_token {
822 Some(token) => next_token = Some(token),
823 None => return Ok(items),
824 }
825 }
826 }
827
828 pub async fn suspend(&self, id: &str) -> Result<(), Error> {
830 super::require_valid_identifier("microvmIdentifier", id)?;
831 let call = Call::post_empty("SuspendMicrovm", paths::suspend(id));
832 send_with_retry(self.transport(), call).await?;
833 Ok(())
834 }
835
836 pub async fn resume(&self, id: &str) -> Result<(), Error> {
855 super::require_valid_identifier("microvmIdentifier", id)?;
856 let call = Call::post_empty("ResumeMicrovm", paths::resume(id));
857 match send_with_retry(self.transport(), call).await {
858 Ok(_) => Ok(()),
859 Err(refused) if refused.kind() == ErrorKind::Platform => {
860 Err(self.ended_before_resume(id, refused).await)
861 }
862 Err(error) => Err(error),
863 }
864 }
865
866 async fn ended_before_resume(&self, id: &str, refused: Error) -> Error {
870 let Ok(vm) = self.get_microvm(id).await else {
871 return refused;
872 };
873 if !crate::constants::DEAD_STATES.contains(&vm.state.as_str()) {
874 return refused;
875 }
876 Error::new(
877 ErrorKind::WindowClosed,
878 format!(
879 "microvm {id} can't be resumed: the service refused ResumeMicrovm, and \
880 GetMicrovm reports the VM {} ({}). A VM that has ended doesn't come back; \
881 launch a new one.",
882 vm.state,
883 vm.reason(),
884 ),
885 )
886 .with_source(refused)
887 }
888
889 pub async fn terminate(&self, id: &str) -> Result<(), Error> {
897 super::require_valid_identifier("microvmIdentifier", id)?;
898 let call = Call::delete("TerminateMicrovm", paths::microvm(id));
899 send_with_retry(self.transport(), call).await?;
900 Ok(())
901 }
902
903 pub async fn mint_auth_token(&self, id: &str) -> Result<ProxyToken, Error> {
915 self.mint_auth_token_for(id, &[ops::PortSpecification::port(self.port())])
916 .await
917 }
918
919 pub async fn mint_auth_token_for(
932 &self,
933 id: &str,
934 ports: &[ops::PortSpecification],
935 ) -> Result<ProxyToken, Error> {
936 super::require_valid_identifier("microvmIdentifier", id)?;
937 if ports.is_empty() {
938 return Err(Error::new(
942 ErrorKind::Precondition,
943 "a proxy token must allow at least one port ('allowedPorts' declares min: 1). \
944 Name the ports this token is for — the agent port plus any workload port \
945 reached through the endpoint.",
946 ));
947 }
948 for spec in ports {
956 match spec {
957 ops::PortSpecification::One { port } => {
958 super::require_valid_port("allowedPorts[].port", *port)?;
959 }
960 ops::PortSpecification::Range { range } => {
961 super::require_valid_port("allowedPorts[].range.startPort", range.start_port)?;
962 super::require_valid_port("allowedPorts[].range.endPort", range.end_port)?;
963 }
964 ops::PortSpecification::All { .. } => {}
966 }
967 }
968 let wire = ops::CreateAuthTokenWire {
969 expiration_in_minutes: MAX_TOKEN_MINUTES,
970 allowed_ports: ports.to_vec(),
971 };
972 let call = Call::post_json("CreateMicrovmAuthToken", paths::auth_token(id), &wire)?;
973 let reply = send_with_retry(self.transport(), call).await?;
974 let minted: ops::CreateAuthTokenResponseWire = reply.json("CreateMicrovmAuthToken")?;
975 ProxyToken::from_map(&minted.auth_token, self.port())
976 }
977
978 pub async fn mint_shell_auth_token(&self, id: &str) -> Result<ProxyToken, Error> {
1000 super::require_valid_identifier("microvmIdentifier", id)?;
1001 let wire = ops::CreateShellAuthTokenWire {
1002 expiration_in_minutes: MAX_TOKEN_MINUTES,
1003 };
1004 let call = Call::post_json(
1005 "CreateMicrovmShellAuthToken",
1006 paths::shell_auth_token(id),
1007 &wire,
1008 )?;
1009 let reply = send_with_retry(self.transport(), call).await?;
1010 let minted: ops::CreateShellAuthTokenResponseWire =
1011 reply.json("CreateMicrovmShellAuthToken")?;
1012 ProxyToken::from_operation_map(
1013 "CreateMicrovmShellAuthToken",
1014 &minted.auth_token,
1015 self.port(),
1016 )
1017 }
1018}
1019
1020pub const SUSPEND_WANTED: [&str; 2] = ["SUSPENDED", "TERMINATED"];
1025
1026#[cfg(test)]
1027mod tests {
1028 use std::sync::Arc;
1029
1030 use super::super::fake::{self as fake, Answer, FakeControlPlane, TestClock};
1031 use super::*;
1032 use crate::region::Region;
1033
1034 #[test]
1038 fn every_state_a_suspend_waits_for_is_in_the_models_enum() {
1039 for state in SUSPEND_WANTED {
1040 assert!(
1041 crate::constants::MICROVM_STATES.contains(&state),
1042 "SUSPEND_WANTED has {state}, which is not a MicrovmState: a suspend would wait \
1043 for a state the service can't report and time out"
1044 );
1045 }
1046 }
1047
1048 fn planted() -> (ControlPlane, Arc<FakeControlPlane>, Arc<TestClock>) {
1049 let fake = Arc::new(FakeControlPlane::new());
1050 let clock = Arc::new(TestClock::new());
1051 let plane = crate::testing::control_plane(fake.clone(), Region::UsEast1, clock.clone());
1052 (plane, fake, clock)
1053 }
1054
1055 #[test]
1062 fn the_payload_ceiling_is_four_thousand_ninety_six_bytes_inclusive() {
1063 let at_ceiling = "a".repeat(4096);
1064 let payload = RunHookPayload::new(at_ceiling.clone()).expect("4096 bytes fits");
1065 assert_eq!(payload.len(), 4096);
1066 assert_eq!(payload.as_str(), at_ceiling);
1067
1068 let over = "a".repeat(4097);
1069 let error = RunHookPayload::new(over).expect_err("4097 bytes does not fit");
1070 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1071 let message = error.to_string();
1072 assert!(message.contains("4097 bytes"), "{message}");
1073 assert!(
1074 message.contains("ceiling of 4096"),
1075 "the message must name the service-model ceiling: {message}"
1076 );
1077 assert!(message.contains("2025-09-09"), "{message}");
1078 assert!(
1079 message.contains("wrong by 4x"),
1080 "the 16 KB claim has to be named as wrong: {message}"
1081 );
1082 }
1083
1084 #[tokio::test]
1087 async fn an_over_ceiling_payload_reaches_no_control_plane_call() {
1088 let (plane, fake, _) = planted();
1089
1090 let refused = RunHookPayload::new("a".repeat(4097));
1092 assert!(refused.is_err());
1093 assert_eq!(fake.calls().len(), 0);
1094
1095 fake.answer(
1098 "RunMicrovm",
1099 Answer::ok(fake::microvm_response("PENDING", None)),
1100 );
1101 let payload = RunHookPayload::new("a".repeat(4096)).expect("fits");
1102 plane
1103 .run_microvm(RunMicrovmRequest::new("arn:image", payload))
1104 .await
1105 .expect("launches");
1106 assert_eq!(fake.call_count("RunMicrovm"), 1);
1107 }
1108
1109 #[test]
1112 fn the_ceiling_counts_utf8_bytes_rather_than_characters() {
1113 let two_byte_chars = "é".repeat(2049);
1114 assert_eq!(
1115 two_byte_chars.chars().count(),
1116 2049,
1117 "well under 4096 chars"
1118 );
1119 assert_eq!(two_byte_chars.len(), 4098, "but over 4096 bytes");
1120
1121 let error = RunHookPayload::new(two_byte_chars).expect_err("bytes are what count");
1122 assert!(error.to_string().contains("4098 bytes"), "{error}");
1123
1124 let exact = "é".repeat(2048);
1126 assert_eq!(exact.len(), 4096);
1127 RunHookPayload::new(exact).expect("exactly at the ceiling");
1128 }
1129
1130 #[test]
1139 fn the_identity_material_costs_its_declared_budget_and_absence_costs_nothing() {
1140 let env = std::collections::HashMap::new();
1141 let without = RunHookPayload::for_launch("tok", &env).expect("fits");
1142 let none_asked = RunHookPayload::for_launch_with_identity("tok", &env, None).expect("fits");
1143 assert_eq!(
1144 without.as_str(),
1145 none_asked.as_str(),
1146 "no identity must produce byte-for-byte the payload this client always sent"
1147 );
1148
1149 let identity = crate::identity::LaunchIdentity::from_seeds([7_u8; 32], [9_u8; 32])
1150 .expect("valid seeds");
1151 let with =
1152 RunHookPayload::for_launch_with_identity("tok", &env, Some(&identity)).expect("fits");
1153 assert_eq!(
1154 with.len() - without.len(),
1155 protocol::identity::IDENTITY_PAYLOAD_BYTES,
1156 "the budget constant must equal what the material really costs on the wire"
1157 );
1158
1159 let parsed = protocol::hook::RunHook::parse(with.as_str()).expect("the daemon can read it");
1161 assert_eq!(
1162 parsed.identity_seed.as_deref(),
1163 Some(identity.seed_field().as_str())
1164 );
1165 assert_eq!(
1166 parsed.identity_host_public_key.as_deref(),
1167 Some(identity.host_public_field().as_str())
1168 );
1169 }
1170
1171 #[test]
1174 fn the_agent_token_payload_is_checked_even_though_the_client_builds_it() {
1175 let payload = RunHookPayload::for_agent_token("bearer-token-value").expect("fits");
1176 assert_eq!(payload.as_str(), r#"{"agent_token":"bearer-token-value"}"#);
1177
1178 let huge = "x".repeat(4096);
1181 let error = RunHookPayload::for_agent_token(&huge)
1182 .expect_err("the JSON wrapper pushes it over the ceiling");
1183 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1184 assert!(
1185 error.to_string().contains("one bearer token fits"),
1186 "{error}"
1187 );
1188 }
1189
1190 #[test]
1196 fn an_empty_launch_env_emits_the_payload_the_token_only_constructor_did() {
1197 let with_empty = RunHookPayload::for_launch("tok", &std::collections::HashMap::new())
1198 .expect("an empty env fits");
1199 let token_only = RunHookPayload::for_agent_token("tok").expect("fits");
1200 assert_eq!(with_empty.as_str(), token_only.as_str());
1201 assert_eq!(with_empty.as_str(), r#"{"agent_token":"tok"}"#);
1202 assert!(
1203 !with_empty.as_str().contains("env"),
1204 "an empty env must not appear on the wire at all: {}",
1205 with_empty.as_str()
1206 );
1207 }
1208
1209 #[test]
1211 fn a_launch_env_rides_in_the_payload_the_daemon_parses() {
1212 let mut env = std::collections::HashMap::new();
1213 env.insert("ANTHROPIC_BASE_URL".to_string(), "https://x".to_string());
1214 let payload = RunHookPayload::for_launch("tok", &env).expect("fits");
1215
1216 let value: serde_json::Value =
1218 serde_json::from_str(payload.as_str()).expect("the payload is JSON");
1219 assert_eq!(value["agent_token"], "tok");
1220 assert_eq!(value["env"]["ANTHROPIC_BASE_URL"], "https://x");
1221 }
1222
1223 #[test]
1241 fn an_over_ceiling_launch_env_is_refused_locally_with_the_byte_count() {
1242 let mut env = std::collections::HashMap::new();
1243 env.insert("CREDENTIALS".to_string(), "c".repeat(4096));
1245
1246 let expected = serde_json::json!({ "agent_token": "tok", "env": &env })
1250 .to_string()
1251 .len();
1252 assert!(
1253 expected > 4096,
1254 "the fixture must actually be over: {expected}"
1255 );
1256
1257 let error = RunHookPayload::for_launch("tok", &env)
1258 .expect_err("a credential-scale launch env does not fit");
1259 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1260 let message = error.to_string();
1261
1262 assert!(
1263 message.contains("ceiling of 4096"),
1264 "the ceiling has to be named: {message}"
1265 );
1266 assert!(
1267 message.contains(&format!("{expected} bytes")),
1268 "the actual byte count has to be named: {message}"
1269 );
1270 assert!(
1271 message.contains("launch env contributed"),
1272 "the env's share of the budget is the actionable half: {message}"
1273 );
1274 assert!(
1275 message.contains("1 variable(s)"),
1276 "the variable count has to be named: {message}"
1277 );
1278 }
1279
1280 #[test]
1287 fn the_launch_env_ceiling_counts_the_serialized_payload_and_not_the_raw_values() {
1288 let mut env = std::collections::HashMap::new();
1289 env.insert("A".to_string(), "a".repeat(2030));
1290 env.insert("B".to_string(), "b".repeat(2030));
1291 let raw: usize = env.values().map(String::len).sum();
1292 assert_eq!(raw, 4060, "the raw values are under the ceiling");
1293 assert!(
1294 raw <= crate::constants::MAX_RUN_HOOK_PAYLOAD_BYTES,
1295 "the fixture only means something if the raw sum fits: {raw}"
1296 );
1297
1298 let error = RunHookPayload::for_launch("tok", &env)
1299 .expect_err("the serialized payload is over even though the raw values are not");
1300 assert!(error.to_string().contains("2 variable(s)"), "{error}");
1301
1302 let mut fits = std::collections::HashMap::new();
1305 fits.insert("A".to_string(), "a".repeat(2000));
1306 fits.insert("B".to_string(), "b".repeat(2000));
1307 let payload = RunHookPayload::for_launch("tok", &fits).expect("4000-odd bytes fits");
1308 assert!(payload.len() <= 4096, "{}", payload.len());
1309 }
1310
1311 #[test]
1313 fn an_empty_payload_is_legal() {
1314 let payload = RunHookPayload::new("").expect("min is 0");
1315 assert!(payload.is_empty());
1316 assert_eq!(payload.len(), 0);
1317 }
1318
1319 #[tokio::test]
1322 async fn the_launch_emits_derived_connector_arns_in_the_right_members() {
1323 let (plane, fake, _) = planted();
1324 fake.answer(
1325 "RunMicrovm",
1326 Answer::ok(fake::microvm_response("PENDING", None)),
1327 );
1328
1329 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1330 let request = RunMicrovmRequest::new("arn:image", payload).with_egress();
1331 plane.run_microvm(request).await.expect("launches");
1332
1333 let calls = fake.calls();
1334 assert_eq!(calls[0].path, "/2025-09-09/microvms");
1335 assert_eq!(calls[0].method, super::super::transport::Method::Post);
1336
1337 let body = fake.first_body("RunMicrovm");
1338 assert_eq!(
1339 body["ingressNetworkConnectors"],
1340 serde_json::json!([
1341 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:ALL_INGRESS"
1342 ])
1343 );
1344 assert_eq!(
1345 body["egressNetworkConnectors"],
1346 serde_json::json!([
1347 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS"
1348 ])
1349 );
1350 assert_eq!(body["runHookPayload"], r#"{"agent_token":"token"}"#);
1351 assert_eq!(body["maximumDurationInSeconds"], 3600);
1352 }
1353
1354 #[tokio::test]
1359 async fn a_launch_without_egress_omits_the_member_entirely() {
1360 let (plane, fake, _) = planted();
1361 fake.answer(
1362 "RunMicrovm",
1363 Answer::ok(fake::microvm_response("PENDING", None)),
1364 );
1365 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1366 plane
1367 .run_microvm(RunMicrovmRequest::new("arn:image", payload))
1368 .await
1369 .expect("launches");
1370
1371 let body = fake.first_body("RunMicrovm");
1372 assert!(
1373 body.get("egressNetworkConnectors").is_none(),
1374 "omitting egress is the whole mechanism: {body}"
1375 );
1376 assert!(body.get("ingressNetworkConnectors").is_some());
1377 }
1378
1379 #[tokio::test]
1380 async fn customer_vpc_connector_arns_reach_the_egress_member_unchanged() {
1381 let (plane, fake, _) = planted();
1382 fake.answer(
1383 "RunMicrovm",
1384 Answer::ok(fake::microvm_response("PENDING", None)),
1385 );
1386 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1387 let mut request = RunMicrovmRequest::new("arn:image", payload);
1388 let arns: Vec<String> = (0..10)
1389 .map(|id| format!("arn:aws:lambda:us-east-1:123456789012:network-connector:vpc-{id}:1"))
1390 .collect();
1391 request.egress_network_connectors = arns.clone();
1392 plane
1393 .run_microvm(request)
1394 .await
1395 .expect("ten egress connectors plus ingress fits");
1396 let body = fake.first_body("RunMicrovm");
1397 assert_eq!(body["egressNetworkConnectors"], serde_json::json!(arns));
1398 assert_eq!(
1399 body["ingressNetworkConnectors"].as_array().unwrap().len(),
1400 1
1401 );
1402 }
1403
1404 #[tokio::test]
1405 async fn invalid_customer_egress_connectors_are_refused_before_launch() {
1406 let valid = "arn:aws:lambda:us-east-1:123456789012:network-connector:vpc-test";
1407 let invalid = [
1408 "VPC_EGRESS",
1409 "arn:aws:lambda:us-west-2:123456789012:network-connector:vpc-test",
1410 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS",
1411 "arn:aws:lambda:us-east-1:123456789012:network-connector:",
1412 "arn:aws:lambda:us-east-1:123456789012:network-connector:vpc-test:0",
1413 "arn:aws:lambda:us-east-1:123456789012:network-connector:vpc-test/other",
1414 ];
1415 for arn in invalid {
1416 let (plane, fake, _) = planted();
1417 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1418 let error = plane
1419 .run_microvm(
1420 RunMicrovmRequest::new("arn:image", payload).with_egress_network_connector(arn),
1421 )
1422 .await
1423 .expect_err(arn);
1424 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1425 assert!(
1426 error.to_string().contains("egressNetworkConnectors"),
1427 "{error}"
1428 );
1429 assert!(fake.calls().is_empty());
1430 }
1431 for (internet, count) in [(true, 1), (false, 11)] {
1432 let (plane, fake, _) = planted();
1433 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1434 let mut request = RunMicrovmRequest::new("arn:image", payload);
1435 request.egress_network_connectors = vec![valid.to_string(); count];
1436 if internet {
1437 request = request.with_egress();
1438 }
1439 let error = plane
1440 .run_microvm(request)
1441 .await
1442 .expect_err("invalid routing request");
1443 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1444 assert!(fake.calls().is_empty());
1445 }
1446 }
1447
1448 #[tokio::test]
1461 async fn a_pinned_launch_version_reaches_the_wire_and_an_unpinned_one_omits_the_member() {
1462 let (plane, fake, _) = planted();
1463 fake.answer(
1464 "RunMicrovm",
1465 Answer::ok(fake::microvm_response("PENDING", None)),
1466 );
1467 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1468 plane
1469 .run_microvm(RunMicrovmRequest::new("arn:image", payload).with_image_version("2.0"))
1470 .await
1471 .expect("launches");
1472
1473 let body = fake.first_body("RunMicrovm");
1474 assert_eq!(body["imageVersion"], "2.0");
1475 assert_eq!(
1476 body["imageIdentifier"], "arn:image",
1477 "pinning a version does not replace the identifier; both are sent"
1478 );
1479
1480 let (plane, fake, _) = planted();
1481 fake.answer(
1482 "RunMicrovm",
1483 Answer::ok(fake::microvm_response("PENDING", None)),
1484 );
1485 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1486 plane
1487 .run_microvm(RunMicrovmRequest::new("arn:image", payload))
1488 .await
1489 .expect("launches");
1490 assert!(
1491 fake.first_body("RunMicrovm").get("imageVersion").is_none(),
1492 "an unpinned launch must send what this client always sent: {}",
1493 fake.first_body("RunMicrovm")
1494 );
1495 }
1496
1497 #[tokio::test]
1507 async fn an_invalid_launch_version_reaches_no_control_plane_call() {
1508 for bad in ["", "2.0\n", "a b"] {
1509 let (plane, fake, _) = planted();
1510 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1511 let error = plane
1512 .run_microvm(RunMicrovmRequest::new("arn:image", payload).with_image_version(bad))
1513 .await
1514 .expect_err(&format!("{bad:?} is not a legal Version"));
1515 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{bad:?}");
1516 assert!(
1517 error.to_string().contains("imageVersion"),
1518 "{bad:?}: {error}"
1519 );
1520 assert_eq!(fake.calls().len(), 0, "{bad:?} must not reach the wire");
1521 }
1522 }
1523
1524 #[tokio::test]
1528 async fn no_bare_connector_name_reaches_the_wire() {
1529 let (plane, fake, _) = planted();
1530 fake.answer(
1531 "RunMicrovm",
1532 Answer::ok(fake::microvm_response("PENDING", None)),
1533 );
1534 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1535 plane
1536 .run_microvm(RunMicrovmRequest::new("arn:image", payload).with_egress())
1537 .await
1538 .expect("launches");
1539
1540 let body = fake.first_body("RunMicrovm");
1541 for member in ["ingressNetworkConnectors", "egressNetworkConnectors"] {
1542 for value in body[member].as_array().expect("a list") {
1543 let text = value.as_str().expect("a string");
1544 assert!(text.starts_with("arn:aws:lambda:"), "{member}: {text}");
1545 assert_ne!(text, "ALL_INGRESS");
1546 assert_ne!(text, "INTERNET_EGRESS");
1547 }
1548 }
1549 }
1550
1551 #[tokio::test]
1553 async fn an_out_of_range_duration_reaches_no_control_plane_call() {
1554 let (plane, fake, _) = planted();
1555 let payload = RunHookPayload::for_agent_token("token").expect("fits");
1556 let mut request = RunMicrovmRequest::new("arn:image", payload);
1557 request.max_duration_sec = 28_801;
1558
1559 let error = plane
1560 .run_microvm(request)
1561 .await
1562 .expect_err("over the ceiling");
1563 assert_eq!(error.kind(), ErrorKind::InvalidArg);
1564 assert_eq!(fake.calls().len(), 0);
1565 }
1566
1567 #[tokio::test]
1577 async fn a_terminal_state_before_running_carries_both_the_state_and_the_reason() {
1578 let (plane, fake, _) = planted();
1579 fake.answer(
1580 "GetMicrovm",
1581 Answer::ok(fake::microvm_response(
1582 "TERMINATED",
1583 Some("run hook returned 500"),
1584 )),
1585 );
1586
1587 let error = plane
1588 .wait_for_running("mvm-abc123", WaitOpts::for_launch())
1589 .await
1590 .expect_err("a dead VM must not be waited out");
1591
1592 assert_eq!(error.kind(), ErrorKind::LaunchDied);
1593 assert_eq!(error.code(), "ERR_LAUNCH_DIED");
1594 let message = error.to_string();
1595 assert!(
1596 message.contains("TERMINATED"),
1597 "the state must be attached: {message}"
1598 );
1599 assert!(
1600 message.contains("run hook returned 500"),
1601 "the stateReason must be attached: {message}"
1602 );
1603 assert!(
1604 message.contains("lifecycle hook failed"),
1605 "the likely cause is worth naming: {message}"
1606 );
1607 assert!(
1608 fake.call_count("GetMicrovm") == 1,
1609 "it fails fast rather than polling to the deadline"
1610 );
1611 }
1612
1613 #[tokio::test]
1620 async fn every_terminal_state_fails_a_launch_fast() {
1621 assert_eq!(crate::constants::TERMINAL_STATES.len(), 4);
1622 for state in crate::constants::TERMINAL_STATES {
1623 let (plane, fake, _) = planted();
1624 fake.answer(
1625 "GetMicrovm",
1626 Answer::ok(fake::microvm_response(state, Some("hook timeout"))),
1627 );
1628 let error = plane
1629 .wait_for_running("mvm-abc123", WaitOpts::for_launch())
1630 .await
1631 .expect_err(&format!("{state} before RUNNING is a death"));
1632 assert_eq!(error.kind(), ErrorKind::LaunchDied, "{state}");
1633 assert!(error.to_string().contains(state), "{state}: {error}");
1634 assert!(
1635 error.to_string().contains("hook timeout"),
1636 "{state}: {error}"
1637 );
1638 assert_eq!(
1639 fake.call_count("GetMicrovm"),
1640 1,
1641 "{state} must fail fast rather than polling to the deadline"
1642 );
1643 }
1644 }
1645
1646 #[tokio::test]
1650 async fn a_missing_state_reason_is_named_rather_than_printed_empty() {
1651 let (plane, fake, _) = planted();
1652 fake.answer(
1653 "GetMicrovm",
1654 Answer::ok(fake::microvm_response("TERMINATED", None)),
1655 );
1656
1657 let error = plane
1658 .wait_for_running("mvm-abc123", WaitOpts::for_launch())
1659 .await
1660 .expect_err("still a death");
1661 let message = error.to_string();
1662 assert!(message.contains("no stateReason"), "{message}");
1663 assert!(
1667 !message.contains(r#"["RUNNING"]: ."#),
1668 "an absent reason must not render as an empty string: {message}"
1669 );
1670 assert_eq!(error.kind(), ErrorKind::LaunchDied);
1671 }
1672
1673 #[tokio::test]
1675 async fn the_launch_wait_polls_through_pending_and_returns_on_running() {
1676 let (plane, fake, _) = planted();
1677 fake.answer(
1678 "GetMicrovm",
1679 Answer::ok(fake::microvm_response("PENDING", None)),
1680 )
1681 .answer(
1682 "GetMicrovm",
1683 Answer::ok(fake::microvm_response("RUNNING", None)),
1684 );
1685
1686 let vm = plane
1687 .wait_for_running("mvm-abc123", WaitOpts::for_launch())
1688 .await
1689 .expect("reaches RUNNING");
1690 assert_eq!(vm.state, "RUNNING");
1691 assert_eq!(
1692 vm.endpoint,
1693 "https://mvm-abc123.microvm.us-east-1.amazonaws.com"
1694 );
1695 assert_eq!(fake.call_count("GetMicrovm"), 2);
1696 }
1697
1698 #[tokio::test]
1702 async fn a_launch_that_never_comes_up_times_out_rather_than_reporting_a_death() {
1703 let (plane, fake, _) = planted();
1704 fake.answer(
1705 "GetMicrovm",
1706 Answer::ok(fake::microvm_response("PENDING", None)),
1707 );
1708
1709 let error = plane
1710 .wait_for_running("mvm-abc123", WaitOpts::for_launch())
1711 .await
1712 .expect_err("the deadline elapses");
1713 assert_eq!(error.kind(), ErrorKind::Timeout);
1714 assert_ne!(
1715 error.kind(),
1716 ErrorKind::LaunchDied,
1717 "a timeout is not a death, and conflating them is what hid the cause"
1718 );
1719 assert!(error.to_string().contains("PENDING"), "{error}");
1720 }
1721
1722 #[tokio::test]
1725 async fn the_resume_wait_tolerates_suspended_and_fails_on_terminated() {
1726 let (plane, fake, _) = planted();
1727 fake.answer(
1728 "GetMicrovm",
1729 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1730 )
1731 .answer(
1732 "GetMicrovm",
1733 Answer::ok(fake::microvm_response("RUNNING", None)),
1734 );
1735
1736 let vm = plane
1737 .wait_for_state(
1738 "mvm-abc123",
1739 &["RUNNING"],
1740 &crate::constants::DEAD_STATES,
1741 WaitOpts::for_launch(),
1742 )
1743 .await
1744 .expect("SUSPENDED is a waypoint on the resume path");
1745 assert_eq!(vm.state, "RUNNING");
1746
1747 let (plane, fake, _) = planted();
1748 fake.answer(
1749 "GetMicrovm",
1750 Answer::ok(fake::microvm_response(
1751 "TERMINATED",
1752 Some("suspended window elapsed"),
1753 )),
1754 );
1755 let error = plane
1756 .wait_for_state(
1757 "mvm-abc123",
1758 &["RUNNING"],
1759 &crate::constants::DEAD_STATES,
1760 WaitOpts::for_launch(),
1761 )
1762 .await
1763 .expect_err("a terminated VM never reaches RUNNING");
1764 assert_eq!(error.kind(), ErrorKind::LaunchDied);
1765 assert!(
1766 error.to_string().contains("suspended window elapsed"),
1767 "{error}"
1768 );
1769 }
1770
1771 #[tokio::test]
1774 async fn a_suspend_treats_terminated_as_a_state_to_report() {
1775 let (plane, fake, _) = planted();
1776 fake.answer(
1777 "GetMicrovm",
1778 Answer::ok(fake::microvm_response("TERMINATED", None)),
1779 );
1780
1781 let vm = plane
1782 .wait_for_state("mvm-abc123", &SUSPEND_WANTED, &[], WaitOpts::for_launch())
1783 .await
1784 .expect("a suspend reports the state it reached");
1785 assert_eq!(vm.state, "TERMINATED");
1786 }
1787
1788 #[tokio::test]
1791 async fn the_lifecycle_calls_use_the_models_paths_and_methods() {
1792 let (plane, fake, _) = planted();
1793 fake.answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
1794 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
1795 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()));
1796
1797 plane.suspend("mvm-1").await.expect("suspends");
1798 plane.resume("mvm-1").await.expect("resumes");
1799 plane.terminate("mvm-1").await.expect("terminates");
1800
1801 let calls = fake.calls();
1802 use super::super::transport::Method;
1803 assert_eq!(
1804 calls
1805 .iter()
1806 .map(|call| (call.operation, call.method, call.path.as_str()))
1807 .collect::<Vec<_>>(),
1808 [
1809 (
1810 "SuspendMicrovm",
1811 Method::Post,
1812 "/2025-09-09/microvms/mvm-1/suspend"
1813 ),
1814 (
1815 "ResumeMicrovm",
1816 Method::Post,
1817 "/2025-09-09/microvms/mvm-1/resume"
1818 ),
1819 (
1820 "TerminateMicrovm",
1821 Method::Delete,
1822 "/2025-09-09/microvms/mvm-1"
1823 ),
1824 ]
1825 );
1826 }
1827
1828 fn terminated_refusal() -> Answer {
1831 Answer {
1832 status: 400,
1833 body: r#"{"message": "The MicroVM mvm-abc123 has been terminated and its state cannot be changed."}"#
1834 .to_string(),
1835 error_type: Some("ValidationException".to_string()),
1836 }
1837 }
1838
1839 #[tokio::test]
1848 async fn a_resume_refused_for_a_vm_that_terminated_while_suspended_is_window_closed() {
1849 for state in crate::constants::DEAD_STATES {
1850 let (plane, fake, _) = planted();
1851 fake.answer("ResumeMicrovm", terminated_refusal()).answer(
1852 "GetMicrovm",
1853 Answer::ok(fake::microvm_response(
1854 state,
1855 Some("MicroVM exceeded maximum lifetime."),
1856 )),
1857 );
1858
1859 let error = plane
1860 .resume("mvm-abc123")
1861 .await
1862 .expect_err("the VM is gone");
1863
1864 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{state}: {error}");
1865 let message = error.to_string();
1866 assert!(message.contains(state), "{state}: {message}");
1867 assert!(
1868 message.contains("MicroVM exceeded maximum lifetime."),
1869 "{state}: the stateReason: {message}"
1870 );
1871 assert_eq!(
1872 fake.operations(),
1873 ["ResumeMicrovm", "GetMicrovm"],
1874 "{state}: one refused resume, then one read"
1875 );
1876 }
1877 }
1878
1879 #[tokio::test]
1886 async fn a_resume_refused_for_a_vm_a_terminate_ended_names_no_cause_and_keeps_the_refusal() {
1887 let (plane, fake, _) = planted();
1888 fake.answer("ResumeMicrovm", terminated_refusal()).answer(
1889 "GetMicrovm",
1890 Answer::ok(fake::microvm_response("TERMINATED", Some("Success."))),
1891 );
1892
1893 let error = plane
1894 .resume("mvm-abc123")
1895 .await
1896 .expect_err("the VM is gone");
1897
1898 assert_eq!(error.kind(), ErrorKind::WindowClosed, "{error}");
1899 let message = error.to_string();
1900 assert_eq!(
1901 message,
1902 "microvm mvm-abc123 can't be resumed: the service refused ResumeMicrovm, and \
1903 GetMicrovm reports the VM TERMINATED (Success.). A VM that has ended doesn't come \
1904 back; launch a new one."
1905 );
1906 assert!(!message.contains(".."), "{message}");
1907 assert!(
1908 !message.contains("suspended"),
1909 "no cause it can't know: {message}"
1910 );
1911
1912 let source = std::error::Error::source(&error).expect("the refusal is the source");
1913 let refused = source
1914 .downcast_ref::<Error>()
1915 .expect("the source is the service's refusal");
1916 assert_eq!(refused.kind(), ErrorKind::Platform);
1917 assert!(
1918 refused
1919 .to_string()
1920 .contains("has been terminated and its state cannot be changed"),
1921 "the service's own text: {refused}"
1922 );
1923 }
1924
1925 #[tokio::test]
1928 async fn a_resume_refused_for_a_live_or_unreadable_vm_keeps_the_services_answer() {
1929 let (plane, fake, _) = planted();
1930 fake.answer(
1931 "ResumeMicrovm",
1932 Answer::failure(409, "The MicroVM is changing state."),
1933 )
1934 .answer(
1935 "GetMicrovm",
1936 Answer::ok(fake::microvm_response("SUSPENDED", None)),
1937 );
1938 let error = plane
1939 .resume("mvm-abc123")
1940 .await
1941 .expect_err("the service refused");
1942 assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
1943 assert!(error.to_string().contains("ConflictException"), "{error}");
1944
1945 let (plane, fake, _) = planted();
1946 fake.answer("ResumeMicrovm", terminated_refusal())
1947 .answer("GetMicrovm", Answer::failure(404, "not found"));
1948 let error = plane
1949 .resume("mvm-abc123")
1950 .await
1951 .expect_err("the service refused");
1952 assert_eq!(error.kind(), ErrorKind::Platform, "{error}");
1953 let message = error.to_string();
1954 assert!(
1955 message.contains("ResumeMicrovm failed") && !message.contains("GetMicrovm"),
1956 "the resume's refusal rather than the read's: {message}"
1957 );
1958 }
1959
1960 #[tokio::test]
1963 async fn a_resume_denied_for_its_credentials_reads_no_state() {
1964 let (plane, fake, _) = planted();
1965 fake.answer(
1966 "ResumeMicrovm",
1967 Answer::failure(403, "User is not authorized to perform ResumeMicrovm"),
1968 )
1969 .answer(
1970 "GetMicrovm",
1971 Answer::ok(fake::microvm_response("TERMINATED", None)),
1972 );
1973 let error = plane.resume("mvm-abc123").await.expect_err("denied");
1974 assert_eq!(error.kind(), ErrorKind::Credentials, "{error}");
1975 assert_eq!(fake.call_count("GetMicrovm"), 0);
1976 }
1977
1978 #[tokio::test]
1981 async fn the_minted_token_comes_from_the_map_and_carries_both_headers() {
1982 let (plane, fake, _) = planted();
1983 fake.answer(
1984 "CreateMicrovmAuthToken",
1985 Answer::ok(fake::auth_token_response("opaque-proxy-token")),
1986 );
1987
1988 let token = plane.mint_auth_token("mvm-1").await.expect("mints");
1989 assert_eq!(token.value(), "opaque-proxy-token");
1990 assert_eq!(token.port(), 9000);
1991
1992 let headers = token.headers();
1993 assert_eq!(
1994 headers[0],
1995 ("X-aws-proxy-auth", "opaque-proxy-token".to_string())
1996 );
1997 assert_eq!(
1998 headers[1],
1999 ("X-aws-proxy-port", "9000".to_string()),
2000 "the auth header without the port header is rejected in a way that reads like a \
2001 bad token"
2002 );
2003
2004 let body = fake.first_body("CreateMicrovmAuthToken");
2005 assert_eq!(body["expirationInMinutes"], 60);
2006 assert_eq!(body["allowedPorts"][0]["port"], 9000);
2007 }
2008
2009 #[tokio::test]
2021 async fn the_shell_mint_names_its_own_route_and_sends_no_ports() {
2022 let (plane, fake, _) = planted();
2023 fake.answer(
2024 "CreateMicrovmShellAuthToken",
2025 Answer::ok(fake::auth_token_response("opaque-shell-jwe")),
2026 );
2027
2028 let token = plane
2029 .mint_shell_auth_token("mvm-1")
2030 .await
2031 .expect("mints a shell token");
2032 assert_eq!(token.value(), "opaque-shell-jwe");
2033
2034 let calls = fake.calls();
2035 assert_eq!(calls.len(), 1);
2036 assert_eq!(calls[0].operation, "CreateMicrovmShellAuthToken");
2037 assert_eq!(calls[0].path, "/2025-09-09/microvms/mvm-1/shell-auth-token");
2038
2039 let body = fake.first_body("CreateMicrovmShellAuthToken");
2040 assert_eq!(body["expirationInMinutes"], 60);
2041 assert_eq!(
2042 body.get("allowedPorts"),
2043 None,
2044 "the shell is not a port: the request declares no allowedPorts member"
2045 );
2046 assert_eq!(
2047 body.as_object().map(serde_json::Map::len),
2048 Some(1),
2049 "expirationInMinutes is the body's only member: {body}"
2050 );
2051 }
2052
2053 #[tokio::test]
2057 async fn the_shell_mint_refuses_an_empty_identifier_before_the_wire() {
2058 let (plane, fake, _) = planted();
2059 let error = plane
2060 .mint_shell_auth_token("")
2061 .await
2062 .expect_err("an empty identifier is refused");
2063 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2064 assert_eq!(fake.calls().len(), 0, "nothing may reach the wire");
2065 }
2066
2067 #[test]
2075 fn a_launch_requests_debug_does_not_print_the_agent_token() {
2076 let payload = RunHookPayload::for_agent_token("SECRETTOKEN").expect("fits");
2077
2078 let rendered = format!("{payload:?}");
2079 assert!(
2080 !rendered.contains("SECRETTOKEN"),
2081 "the agent token reached a Debug string: {rendered}"
2082 );
2083 assert!(
2084 rendered.contains(&format!("{} bytes", payload.len())),
2085 "the size is what a TRAP-5 diagnosis needs: {rendered}"
2086 );
2087
2088 let request = RunMicrovmRequest::new("arn:image", payload);
2089 let rendered = format!("{request:?}");
2090 assert!(rendered.contains("arn:image"), "{rendered}");
2091 assert!(
2092 !rendered.contains("SECRETTOKEN"),
2093 "the agent token reached a Debug string through the request: {rendered}"
2094 );
2095 }
2096
2097 #[test]
2104 fn a_proxy_token_debug_does_not_print_the_credential() {
2105 let mut map = std::collections::BTreeMap::new();
2106 map.insert(
2107 PROXY_AUTH_HEADER.to_string(),
2108 "eyJhbGciOi-secret".to_string(),
2109 );
2110 let token = ProxyToken::from_map(&map, 9000).expect("the map has the auth key");
2111
2112 let rendered = format!("{token:?}");
2113 assert!(rendered.contains(PROXY_AUTH_HEADER), "{rendered}");
2114 assert!(rendered.contains("9000"), "{rendered}");
2115 assert!(
2116 !rendered.contains("secret"),
2117 "the token value reached a Debug string: {rendered}"
2118 );
2119 }
2120
2121 #[test]
2124 fn a_map_missing_the_auth_key_is_a_retryable_mint_failure() {
2125 let mut map = std::collections::BTreeMap::new();
2126 map.insert("SomeOtherKey".to_string(), "value".to_string());
2127
2128 let error = ProxyToken::from_map(&map, 9000).expect_err("no auth key");
2129 assert_eq!(
2130 error.wire_kind(),
2131 Some(crate::error::WireKind::AuthTokenMint)
2132 );
2133 assert!(error.retryable(), "minting is inside the retry path");
2134 assert!(
2135 error.to_string().contains("is a map, not a string"),
2136 "{error}"
2137 );
2138 }
2139
2140 #[tokio::test]
2142 async fn the_token_is_minted_for_the_configured_port() {
2143 let fake = Arc::new(FakeControlPlane::new());
2144 let plane = crate::testing::control_plane(
2145 fake.clone(),
2146 Region::UsEast1,
2147 Arc::new(TestClock::new()),
2148 )
2149 .with_port(8080)
2150 .expect("8080 is a legal port");
2151 fake.answer(
2152 "CreateMicrovmAuthToken",
2153 Answer::ok(fake::auth_token_response("t")),
2154 );
2155
2156 let token = plane.mint_auth_token("mvm-1").await.expect("mints");
2157 assert_eq!(token.port(), 8080);
2158 assert_eq!(
2159 fake.first_body("CreateMicrovmAuthToken")["allowedPorts"][0]["port"],
2160 8080
2161 );
2162 }
2163
2164 #[tokio::test]
2188 async fn a_full_lifecycle_never_calls_shell_auth_and_requests_only_what_was_asked() {
2189 let (plane, fake, _) = planted();
2190 fake.answer(
2191 "CreateMicrovmImage",
2192 Answer::created(fake::create_image_response("img")),
2193 )
2194 .answer(
2195 "GetMicrovmImage",
2196 Answer::ok(fake::get_image_response("img", "CREATED")),
2197 )
2198 .answer(
2199 "RunMicrovm",
2200 Answer::ok(fake::microvm_response("PENDING", None)),
2201 )
2202 .answer(
2203 "GetMicrovm",
2204 Answer::ok(fake::microvm_response("RUNNING", None)),
2205 )
2206 .answer(
2207 "CreateMicrovmAuthToken",
2208 Answer::ok(fake::auth_token_response("proxy-token")),
2209 )
2210 .answer("SuspendMicrovm", Answer::ok(fake::empty_response()))
2211 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()))
2212 .answer("TerminateMicrovm", Answer::ok(fake::empty_response()))
2213 .answer(
2214 "ListMicrovmImageVersions",
2215 Answer::ok(fake::list_versions_response("1")),
2216 )
2217 .answer(
2218 "DeleteMicrovmImage",
2219 Answer::ok(fake::delete_image_response()),
2220 );
2221
2222 let request = super::super::CreateImageRequest::new(
2224 "img",
2225 crate::testing::aarch64_daemon(b"binary"),
2226 "s3://bucket/img.zip",
2227 "arn:aws:iam::123456789012:role/build",
2228 );
2229 let image = plane.create_image(request).await.expect("creates");
2230 plane
2231 .wait_for_image(&image.identifier, image.size, WaitOpts::default())
2232 .await
2233 .expect("becomes usable");
2234
2235 let payload = RunHookPayload::for_agent_token("agent-token").expect("fits");
2236 let vm = plane
2237 .run_microvm(
2238 super::super::RunMicrovmRequest::new(&image.identifier, payload).with_egress(),
2239 )
2240 .await
2241 .expect("launches");
2242 plane
2243 .wait_for_running(&vm.id, WaitOpts::for_launch())
2244 .await
2245 .expect("reaches RUNNING");
2246 plane.mint_auth_token(&vm.id).await.expect("mints");
2247 plane.suspend(&vm.id).await.expect("suspends");
2248 plane.resume(&vm.id).await.expect("resumes");
2249 plane.terminate(&vm.id).await.expect("terminates");
2250 plane
2251 .delete_image(&image.identifier, 1, std::time::Duration::from_secs(1))
2252 .await;
2253
2254 let operations = fake.operations();
2257 assert!(
2258 operations.len() >= 10,
2259 "the lifecycle must actually have run: {operations:?}"
2260 );
2261 assert!(operations.contains(&"CreateMicrovmAuthToken"));
2262
2263 assert_eq!(
2264 fake.call_count("CreateMicrovmShellAuthToken"),
2265 0,
2266 "the shell mint exists for `microvm shell` alone, and a lifecycle that asked \
2267 for no shell must mint no shell token: {operations:?}"
2268 );
2269 for operation in &operations {
2270 assert!(
2271 !operation.contains("Shell"),
2272 "no operation may be a shell one: {operation}"
2273 );
2274 }
2275 for path in fake.paths() {
2276 assert!(
2277 !path.contains("shell"),
2278 "no request may reach a shell route: {path}"
2279 );
2280 }
2281
2282 let launch = fake.first_body("RunMicrovm");
2288 assert_eq!(
2289 launch["ingressNetworkConnectors"],
2290 serde_json::json!([
2291 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:ALL_INGRESS"
2292 ]),
2293 "ingress is exactly what the caller asked for"
2294 );
2295 assert_eq!(
2296 launch["egressNetworkConnectors"],
2297 serde_json::json!([
2298 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS"
2299 ]),
2300 "egress is exactly what the caller asked for"
2301 );
2302 }
2303
2304 #[tokio::test]
2310 async fn http_plus_shell_ingress_is_the_pair_that_reaches_the_wire() {
2311 let (plane, fake, _) = planted();
2312 fake.answer(
2313 "RunMicrovm",
2314 Answer::ok(fake::microvm_response("PENDING", None)),
2315 );
2316
2317 let payload = RunHookPayload::for_agent_token("agent-token").expect("fits");
2318 let mut request = RunMicrovmRequest::new("arn:image", payload);
2319 request.connectors = vec![
2320 super::super::ConnectorIntent::HttpIngress,
2321 super::super::ConnectorIntent::ShellIngress,
2322 ];
2323 plane.run_microvm(request).await.expect("launches");
2324
2325 let launch = fake.first_body("RunMicrovm");
2326 assert_eq!(
2327 launch["ingressNetworkConnectors"],
2328 serde_json::json!([
2329 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:HTTP_INGRESS",
2330 "arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:SHELL_INGRESS"
2331 ]),
2332 "both fine-grained ingress connectors reach the wire, in the order asked"
2333 );
2334 assert_eq!(
2335 launch.get("egressNetworkConnectors"),
2336 None,
2337 "no egress was asked for, so the member is absent rather than empty"
2338 );
2339 }
2340
2341 #[tokio::test]
2350 async fn all_ingress_combined_with_shell_ingress_is_refused_before_the_wire() {
2351 let (plane, fake, _) = planted();
2352
2353 let payload = RunHookPayload::for_agent_token("agent-token").expect("fits");
2354 let mut request = RunMicrovmRequest::new("arn:image", payload);
2355 request
2356 .connectors
2357 .push(super::super::ConnectorIntent::ShellIngress);
2358 let error = plane
2359 .run_microvm(request)
2360 .await
2361 .expect_err("the invalid combination is refused locally");
2362
2363 assert!(
2364 error
2365 .to_string()
2366 .contains("HTTP_INGRESS and/or SHELL_INGRESS"),
2367 "the refusal names the pair that works: {error}"
2368 );
2369 assert_eq!(
2370 fake.call_count("RunMicrovm"),
2371 0,
2372 "nothing launched, so nothing billed"
2373 );
2374 }
2375
2376 #[tokio::test]
2387 async fn shell_ingress_without_http_ingress_is_refused_before_the_wire() {
2388 let (plane, fake, _) = planted();
2389
2390 let payload = RunHookPayload::for_agent_token("agent-token").expect("fits");
2391 let mut request = RunMicrovmRequest::new("arn:image", payload);
2392 request.connectors = vec![
2393 super::super::ConnectorIntent::ShellIngress,
2394 super::super::ConnectorIntent::Egress,
2395 ];
2396 let error = plane
2397 .run_microvm(request)
2398 .await
2399 .expect_err("shell ingress without HTTP ingress is refused locally");
2400
2401 assert!(
2402 error.to_string().contains("HTTP_INGRESS"),
2403 "the refusal names the missing connector: {error}"
2404 );
2405 assert_eq!(
2406 fake.call_count("RunMicrovm"),
2407 0,
2408 "nothing launched, so nothing billed"
2409 );
2410 }
2411
2412 #[tokio::test]
2421 async fn the_fleet_listing_retries_a_page_the_service_throttled_as_400() {
2422 let (plane, fake, _) = planted();
2423 fake.answer(
2424 "ListMicrovms",
2425 Answer::ok(fake::list_microvms_page(
2426 &["mvm-page1-a"],
2427 Some("fleet-page-2"),
2428 )),
2429 )
2430 .answer("ListMicrovms", Answer::throttled_as_400())
2431 .answer(
2432 "ListMicrovms",
2433 Answer::ok(fake::list_microvms_page(&["mvm-page2-a"], None)),
2434 );
2435
2436 let fleet = plane
2437 .list_microvms()
2438 .await
2439 .expect("lists past the throttle");
2440 let ids: Vec<&str> = fleet.iter().map(|vm| vm.microvm_id.as_str()).collect();
2441 assert_eq!(ids, ["mvm-page1-a", "mvm-page2-a"]);
2442 assert_eq!(
2443 fake.call_count("ListMicrovms"),
2444 3,
2445 "page two was asked for again after the throttle"
2446 );
2447 let paths = fake.paths();
2448 assert_eq!(paths[1], paths[2], "the retry is the identical request");
2449 }
2450
2451 #[tokio::test]
2464 async fn the_fleet_listing_follows_next_token_to_the_vms_on_page_two() {
2465 let (plane, fake, _) = planted();
2466 fake.answer(
2467 "ListMicrovms",
2468 Answer::ok(fake::list_microvms_page(
2469 &["mvm-page1-a", "mvm-page1-b"],
2470 Some("fleet-page-2"),
2471 )),
2472 )
2473 .answer(
2474 "ListMicrovms",
2475 Answer::ok(fake::list_microvms_page(&["mvm-page2-a"], None)),
2476 );
2477
2478 let fleet = plane.list_microvms().await.expect("lists");
2479 assert_eq!(
2480 fleet.len(),
2481 3,
2482 "a fleet spanning two pages is three VMs, not two"
2483 );
2484
2485 let ids: Vec<&str> = fleet.iter().map(|vm| vm.microvm_id.as_str()).collect();
2486 assert!(ids.contains(&"mvm-page2-a"), "page two is missing: {ids:?}");
2487 assert!(ids.contains(&"mvm-page1-a"), "{ids:?}");
2488 assert_eq!(fake.call_count("ListMicrovms"), 2, "both pages were read");
2489
2490 let paths = fake.paths();
2491 assert!(
2492 paths[1].contains("nextToken=fleet-page-2"),
2493 "the second request must carry the first page's token: {}",
2494 paths[1]
2495 );
2496 assert!(
2497 !paths[0].contains('?'),
2498 "the first request carries no cursor: {}",
2499 paths[0]
2500 );
2501 }
2502
2503 #[tokio::test]
2511 async fn a_vm_the_fleet_listing_meets_on_two_pages_is_listed_once() {
2512 let (plane, fake, _) = planted();
2513 fake.answer(
2514 "ListMicrovms",
2515 Answer::ok(fake::list_microvms_page(
2516 &["mvm-a", "mvm-moved"],
2517 Some("fleet-page-2"),
2518 )),
2519 )
2520 .answer(
2521 "ListMicrovms",
2522 Answer::ok(
2523 fake::list_microvms_page(&["mvm-moved", "mvm-b"], None).replacen(
2524 "RUNNING",
2525 "SUSPENDED",
2526 1,
2527 ),
2528 ),
2529 );
2530
2531 let fleet = plane.list_microvms().await.expect("lists");
2532 let ids: Vec<&str> = fleet.iter().map(|vm| vm.microvm_id.as_str()).collect();
2533 assert_eq!(
2534 ids,
2535 ["mvm-a", "mvm-moved", "mvm-b"],
2536 "each VM once, in the order first listed"
2537 );
2538 assert_eq!(
2539 fleet[1].state, "SUSPENDED",
2540 "the page read later is the newer reading of the VM"
2541 );
2542 assert_eq!(fleet[0].state, "RUNNING");
2543 assert_eq!(fake.call_count("ListMicrovms"), 2, "both pages were read");
2544 }
2545
2546 #[tokio::test]
2556 async fn the_reported_idle_policy_is_carried_rather_than_dropped() {
2557 let (plane, fake, _) = planted();
2558 fake.answer(
2559 "GetMicrovm",
2560 Answer::ok(fake::microvm_response("RUNNING", None)),
2561 );
2562
2563 let vm = plane.get_microvm("mvm-1").await.expect("reads");
2564 let policy = vm
2565 .idle_policy
2566 .expect("GetMicrovm returns an idlePolicy, measured 2026-08-15");
2567 assert_eq!(policy.max_idle_duration_seconds, 1800);
2568 assert_eq!(
2569 policy.suspended_duration_seconds, 600,
2570 "the member the comment claimed was request-only"
2571 );
2572 assert!(!policy.auto_resume_enabled);
2573 }
2574
2575 #[tokio::test]
2579 async fn an_absent_idle_policy_is_none_rather_than_a_zero_window() {
2580 let (plane, fake, _) = planted();
2581 fake.answer(
2582 "GetMicrovm",
2583 Answer::ok(
2584 r#"{
2585 "microvmId": "mvm-1",
2586 "state": "RUNNING",
2587 "endpoint": "https://e",
2588 "imageArn": "arn:aws:lambda:us-east-1:123456789012:microvm-image:img",
2589 "imageVersion": "1"
2590 }"#,
2591 ),
2592 );
2593
2594 let vm = plane.get_microvm("mvm-1").await.expect("parses");
2595 assert_eq!(vm.idle_policy, None);
2596 }
2597
2598 #[tokio::test]
2601 async fn a_transport_failure_is_retried_rather_than_reported() {
2602 let (plane, fake, _) = planted();
2603 fake.fail_transport("GetMicrovm", 2).answer(
2604 "GetMicrovm",
2605 Answer::ok(fake::microvm_response("RUNNING", None)),
2606 );
2607
2608 let vm = plane
2609 .get_microvm("mvm-1")
2610 .await
2611 .expect("the retry succeeds");
2612 assert_eq!(vm.state, "RUNNING");
2613 assert_eq!(
2614 fake.call_count("GetMicrovm"),
2615 3,
2616 "two failures then a success"
2617 );
2618 }
2619
2620 #[tokio::test]
2637 async fn an_under_minimum_idle_duration_reaches_no_control_plane_call() {
2638 for under in [0, 1, 59] {
2639 let (plane, fake, _) = planted();
2640 fake.answer(
2641 "RunMicrovm",
2642 Answer::ok(fake::microvm_response("PENDING", None)),
2643 );
2644
2645 let payload = RunHookPayload::for_agent_token("token").expect("fits");
2646 let mut request = RunMicrovmRequest::new("arn:image", payload);
2647 request.max_idle_sec = under;
2648
2649 let error = plane
2650 .run_microvm(request)
2651 .await
2652 .expect_err("the model's min is 60");
2653 assert_eq!(error.kind(), ErrorKind::InvalidArg);
2654 let message = error.to_string();
2655 assert!(message.contains("maxIdleDurationSeconds"), "{message}");
2656 assert!(message.contains(&under.to_string()), "{message}");
2657 assert_eq!(
2658 fake.calls().len(),
2659 0,
2660 "{under} was refused locally; before this guard it was serialized and sent"
2661 );
2662 }
2663
2664 let (plane, fake, _) = planted();
2667 fake.answer(
2668 "RunMicrovm",
2669 Answer::ok(fake::microvm_response("PENDING", None)),
2670 );
2671 let payload = RunHookPayload::for_agent_token("token").expect("fits");
2672 let mut request = RunMicrovmRequest::new("arn:image", payload);
2673 request.max_idle_sec = 60;
2674 plane.run_microvm(request).await.expect("60 is the minimum");
2675 assert_eq!(
2676 fake.first_body("RunMicrovm")["idlePolicy"]["maxIdleDurationSeconds"],
2677 60
2678 );
2679 }
2680
2681 #[tokio::test]
2691 async fn a_malformed_launch_role_or_identifier_reaches_no_control_plane_call() {
2692 type Mutate = fn(&mut RunMicrovmRequest);
2695 let rows: [(&str, Mutate, &str); 6] = [
2696 (
2697 "an execution role that is a bare name",
2698 |request| {
2699 request.execution_role_arn = Some("execution-role".to_string());
2700 },
2701 "role *name*",
2702 ),
2703 (
2704 "an execution role with eleven account digits",
2705 |request| {
2706 request.execution_role_arn =
2707 Some("arn:aws:iam::12345678901:role/exec".to_string());
2708 },
2709 "exactly twelve digits",
2710 ),
2711 (
2712 "an execution role that is a function ARN",
2713 |request| {
2714 request.execution_role_arn =
2715 Some("arn:aws:lambda:us-east-1:123456789012:function:h".to_string());
2716 },
2717 "RoleArn pattern",
2718 ),
2719 (
2720 "an empty image identifier",
2721 |request| {
2722 request.image_identifier = String::new();
2723 },
2724 "collapses the path",
2725 ),
2726 (
2727 "a 257-character image identifier",
2728 |request| {
2729 request.image_identifier = "a".repeat(257);
2730 },
2731 "MicrovmImageArn permits 2048",
2732 ),
2733 (
2734 "an image version with a newline",
2735 |request| {
2736 request.image_version = Some("2.0\n".to_string());
2737 },
2738 "contains whitespace",
2739 ),
2740 ];
2741
2742 for (label, mutate, expected) in rows {
2743 let (plane, fake, _) = planted();
2744 fake.answer(
2745 "RunMicrovm",
2746 Answer::ok(fake::microvm_response("PENDING", None)),
2747 );
2748
2749 let payload = RunHookPayload::for_agent_token("token").expect("fits");
2750 let mut request = RunMicrovmRequest::new("arn:image", payload);
2751 mutate(&mut request);
2752
2753 let error = plane
2754 .run_microvm(request)
2755 .await
2756 .expect_err(&format!("{label} must be refused"));
2757 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
2758 assert!(
2759 error.to_string().contains(expected),
2760 "{label}: wanted {expected:?}, got {error}"
2761 );
2762 assert_eq!(fake.calls().len(), 0, "{label}");
2763 }
2764
2765 let (plane, fake, _) = planted();
2768 fake.answer(
2769 "RunMicrovm",
2770 Answer::ok(fake::microvm_response("PENDING", None)),
2771 );
2772 let payload = RunHookPayload::for_agent_token("token").expect("fits");
2773 let mut request = RunMicrovmRequest::new(
2774 "arn:aws:lambda:us-east-1:392583147479:microvm-image:agentd-conformance",
2775 payload,
2776 );
2777 request.execution_role_arn =
2778 Some("arn:aws:iam::392583147479:role/bonk-sandbox-microvm-execution".to_string());
2779 request.image_version = Some("1.0".to_string());
2780 plane
2781 .run_microvm(request)
2782 .await
2783 .expect("a realistic pinned launch with a real execution role");
2784 assert_eq!(fake.call_count("RunMicrovm"), 1);
2785 }
2786
2787 #[tokio::test]
2797 async fn no_lifecycle_operation_sends_an_empty_or_over_long_microvm_identifier() {
2798 for bad in ["", &"a".repeat(257)] {
2799 let (plane, fake, _) = planted();
2800 fake.answer(
2803 "GetMicrovm",
2804 Answer::ok(fake::microvm_response("RUNNING", None)),
2805 );
2806 fake.answer("SuspendMicrovm", Answer::ok("{}"));
2807 fake.answer("ResumeMicrovm", Answer::ok("{}"));
2808 fake.answer("TerminateMicrovm", Answer::ok("{}"));
2809 fake.answer(
2810 "CreateMicrovmAuthToken",
2811 Answer::ok(fake::auth_token_response("t")),
2812 );
2813
2814 plane.get_microvm(bad).await.expect_err("GetMicrovm");
2815 plane.suspend(bad).await.expect_err("SuspendMicrovm");
2816 plane.resume(bad).await.expect_err("ResumeMicrovm");
2817 plane.terminate(bad).await.expect_err("TerminateMicrovm");
2818 plane
2819 .mint_auth_token(bad)
2820 .await
2821 .expect_err("CreateMicrovmAuthToken");
2822 plane
2823 .wait_for_running(bad, WaitOpts::for_launch())
2824 .await
2825 .expect_err("GetMicrovm, in a loop");
2826
2827 assert_eq!(
2828 fake.calls().len(),
2829 0,
2830 "six operations refused {bad:?} locally. An empty identifier on the terminate \
2831 path is a DELETE addressed at the collection."
2832 );
2833 }
2834 }
2835
2836 #[tokio::test]
2851 async fn no_proxy_token_is_minted_for_port_zero() {
2852 let rows: [(&str, ops::PortSpecification, &str); 3] = [
2853 (
2854 "a single port of 0",
2855 ops::PortSpecification::port(0),
2856 "allowedPorts[].port",
2857 ),
2858 (
2859 "a range starting at 0",
2860 ops::PortSpecification::range(0, 9000),
2861 "allowedPorts[].range.startPort",
2862 ),
2863 (
2864 "a range ending at 0",
2865 ops::PortSpecification::range(0, 0),
2866 "allowedPorts[].range.startPort",
2867 ),
2868 ];
2869
2870 for (label, spec, member) in rows {
2871 let (plane, fake, _) = planted();
2872 fake.answer(
2873 "CreateMicrovmAuthToken",
2874 Answer::ok(fake::auth_token_response("t")),
2875 );
2876
2877 let error = plane
2878 .mint_auth_token_for("mvm-1", &[spec])
2879 .await
2880 .expect_err(&format!("{label} must be refused"));
2881 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{label}");
2882 let message = error.to_string();
2883 assert!(message.contains(member), "{label}: {message}");
2884 assert!(
2885 message.contains("authorizes nothing"),
2886 "{label}: the 403 consequence has to be in the message: {message}"
2887 );
2888 assert_eq!(fake.calls().len(), 0, "{label}");
2889 }
2890
2891 let (plane, fake, _) = planted();
2895 fake.answer(
2896 "CreateMicrovmAuthToken",
2897 Answer::ok(fake::auth_token_response("t")),
2898 );
2899 plane
2900 .mint_auth_token_for(
2901 "mvm-1",
2902 &[
2903 ops::PortSpecification::port(9000),
2904 ops::PortSpecification::port(0),
2905 ],
2906 )
2907 .await
2908 .expect_err("every entry is checked, not the first");
2909 assert_eq!(fake.calls().len(), 0);
2910
2911 for legal in [
2914 ops::PortSpecification::port(1),
2915 ops::PortSpecification::port(65_535),
2916 ops::PortSpecification::range(1, 65_535),
2917 ops::PortSpecification::all(),
2918 ] {
2919 let (plane, fake, _) = planted();
2920 fake.answer(
2921 "CreateMicrovmAuthToken",
2922 Answer::ok(fake::auth_token_response("t")),
2923 );
2924 plane
2925 .mint_auth_token_for("mvm-1", &[legal])
2926 .await
2927 .expect("a legal port specification mints");
2928 assert_eq!(fake.call_count("CreateMicrovmAuthToken"), 1);
2929 }
2930 }
2931
2932 #[tokio::test]
2938 async fn an_adopted_launch_that_finds_its_vm_suspended_resumes_it() {
2939 for first in ["SUSPENDING", "SUSPENDED"] {
2940 let (plane, fake, _) = planted();
2941 fake.answer(
2942 "GetMicrovm",
2943 Answer::ok(fake::microvm_response(first, None)),
2944 )
2945 .answer(
2946 "GetMicrovm",
2947 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2948 )
2949 .answer(
2950 "GetMicrovm",
2951 Answer::ok(fake::microvm_response("SUSPENDED", None)),
2952 )
2953 .answer(
2954 "GetMicrovm",
2955 Answer::ok(fake::microvm_response("RUNNING", None)),
2956 )
2957 .answer("ResumeMicrovm", Answer::ok(fake::empty_response()));
2958 let running = plane
2959 .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2960 .await
2961 .unwrap_or_else(|error| panic!("{first}: {error}"));
2962 assert_eq!(running.state, "RUNNING", "{first}");
2963 assert_eq!(
2964 fake.call_count("ResumeMicrovm"),
2965 1,
2966 "{first}: one resume, even though GetMicrovm still read SUSPENDED after it"
2967 );
2968 }
2969 }
2970
2971 #[tokio::test]
2973 async fn an_adopted_launch_still_fails_fast_on_a_dead_state() {
2974 for state in crate::constants::DEAD_STATES {
2975 let (plane, fake, _) = planted();
2976 fake.answer(
2977 "GetMicrovm",
2978 Answer::ok(fake::microvm_response(state, Some("hook timeout"))),
2979 );
2980 let error = plane
2981 .wait_for_launch("mvm-abc123", WaitOpts::for_launch())
2982 .await
2983 .expect_err(&format!("{state} is a death"));
2984 assert_eq!(error.kind(), ErrorKind::LaunchDied, "{state}");
2985 assert!(
2986 error.to_string().contains("hook timeout"),
2987 "{state}: {error}"
2988 );
2989 assert_eq!(fake.call_count("GetMicrovm"), 1, "{state}");
2990 assert_eq!(fake.call_count("ResumeMicrovm"), 0, "{state}");
2991 }
2992 }
2993
2994 #[tokio::test]
2996 async fn get_microvm_carries_the_lifetime_members() {
2997 let (plane, fake, _) = planted();
2998 fake.answer(
2999 "GetMicrovm",
3000 Answer::ok(fake::microvm_response("RUNNING", None)),
3001 );
3002 let vm = plane.get_microvm("mvm-abc123").await.expect("parses");
3003 assert_eq!(vm.maximum_duration_seconds, Some(3600));
3004 assert_eq!(
3005 vm.started_at,
3006 Some(std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_754_524_800))
3007 );
3008 assert_eq!(vm.terminated_at, None);
3009 }
3010
3011 #[tokio::test]
3020 async fn an_out_of_range_timestamp_reads_as_absent() {
3021 for seconds in ["2e19", "1e300", "1.7976931348623157e308", "-1"] {
3022 let (plane, fake, _) = planted();
3023 let body = fake::microvm_response("TERMINATED", None).replace(
3024 r#""startedAt": 1754524800"#,
3025 &format!(r#""startedAt": {seconds}, "terminatedAt": {seconds}"#),
3026 );
3027 fake.answer("GetMicrovm", Answer::ok(body));
3028 let vm = plane
3029 .get_microvm("mvm-abc123")
3030 .await
3031 .unwrap_or_else(|error| panic!("{seconds}: {error}"));
3032 assert_eq!(vm.state, "TERMINATED", "{seconds}");
3033 assert_eq!(vm.maximum_duration_seconds, Some(3600), "{seconds}");
3034 assert_eq!(vm.started_at, None, "{seconds}");
3035 assert_eq!(vm.terminated_at, None, "{seconds}");
3036 }
3037 }
3038
3039 #[tokio::test]
3041 async fn list_filters_reach_the_query_string() {
3042 let (plane, fake, _) = planted();
3043 fake.answer(
3044 "ListMicrovms",
3045 Answer::ok(fake::list_microvms_page(&["mvm-1"], None)),
3046 );
3047 let filter = MicrovmFilter {
3048 image_identifier: Some(
3049 "arn:aws:lambda:us-east-1:123456789012:microvm-image:img".into(),
3050 ),
3051 image_version: Some("1.0".into()),
3052 };
3053 let items = plane.list_microvms_matching(&filter).await.expect("lists");
3054 assert_eq!(items.len(), 1);
3055 let path = fake.paths().pop().expect("one call");
3056 assert!(
3057 path.ends_with(
3058 "/microvms?imageIdentifier=arn%3Aaws%3Alambda%3Aus-east-1%3A123456789012%3Amicrovm-image%3Aimg&imageVersion=1.0"
3059 ),
3060 "{path}"
3061 );
3062 plane.list_microvms().await.expect("unfiltered");
3063 assert!(fake.paths().pop().expect("second").ends_with("/microvms"));
3064 }
3065
3066 #[tokio::test]
3068 async fn per_vm_logging_is_sent_and_checked_locally() {
3069 let (plane, fake, _) = planted();
3070 fake.answer(
3071 "RunMicrovm",
3072 Answer::ok(fake::microvm_response("PENDING", None)),
3073 );
3074 let payload = || RunHookPayload::for_agent_token("t").expect("fits");
3075 let mut request = RunMicrovmRequest::new("arn:image", payload());
3076 request.logging = Some(ops::Logging::cloud_watch("/team/agents", None));
3077 plane.run_microvm(request).await.expect("launches");
3078 assert_eq!(
3079 fake.first_body("RunMicrovm")["logging"],
3080 serde_json::json!({"cloudWatch": {"logGroup": "/team/agents"}})
3081 );
3082
3083 let mut disabled = RunMicrovmRequest::new("arn:image", payload());
3084 disabled.logging = Some(ops::Logging::disabled());
3085 plane.run_microvm(disabled).await.expect("launches");
3086 assert_eq!(
3087 fake.body_of("RunMicrovm", 1)["logging"],
3088 serde_json::json!({"disabled": {}})
3089 );
3090
3091 let plain = RunMicrovmRequest::new("arn:image", payload());
3092 plane.run_microvm(plain).await.expect("launches");
3093 assert!(fake.body_of("RunMicrovm", 2).get("logging").is_none());
3094
3095 for bad in [
3096 ops::Logging::cloud_watch("bad group!", None),
3097 ops::Logging::CloudWatch {
3098 cloud_watch: ops::CloudWatchLogging {
3099 log_group: None,
3100 log_stream: Some("s".into()),
3101 },
3102 },
3103 ] {
3104 let mut request = RunMicrovmRequest::new("arn:image", payload());
3105 request.logging = Some(bad);
3106 let error = plane.run_microvm(request).await.expect_err("refused");
3107 assert_eq!(error.kind(), ErrorKind::InvalidArg, "{error}");
3108 }
3109 assert_eq!(
3110 fake.call_count("RunMicrovm"),
3111 3,
3112 "the refusals cost no call"
3113 );
3114 }
3115}