1use crate::error::{Error, ErrorKind};
37use crate::region::Region;
38use std::time::Duration;
39
40#[derive(Clone, Copy, Debug, PartialEq, Eq)]
42pub struct RetryPolicy {
43 max_attempts: u32,
44 max_elapsed: Duration,
45}
46
47impl RetryPolicy {
48 pub fn new(max_attempts: u32, max_elapsed: Duration) -> Result<Self, Error> {
51 if max_attempts == 0 || max_elapsed.is_zero() {
52 return Err(Error::invalid_arg(
53 "retry max_attempts and max_elapsed must be positive",
54 ));
55 }
56 Ok(Self {
57 max_attempts,
58 max_elapsed,
59 })
60 }
61}
62
63#[derive(Clone, Debug)]
65pub struct RetryCancellation(tokio::sync::watch::Sender<bool>);
66
67impl Default for RetryCancellation {
68 fn default() -> Self {
69 Self(tokio::sync::watch::Sender::new(false))
70 }
71}
72
73impl RetryCancellation {
74 pub fn cancel(&self) {
76 self.0.send_replace(true);
77 }
78
79 fn is_cancelled(&self) -> bool {
80 *self.0.borrow()
81 }
82
83 pub(crate) async fn cancelled(&self) {
84 let mut receiver = self.0.subscribe();
85 let _ = receiver.wait_for(|cancelled| *cancelled).await;
86 }
87}
88
89#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
91pub struct RetryDetails {
92 pub attempt_count: u32,
93 pub backoff: Duration,
94}
95
96fn retry_delays() -> backon::ExponentialBuilder {
97 backon::ExponentialBuilder::default()
98 .with_jitter()
99 .with_min_delay(Duration::from_millis(200))
100 .with_max_delay(Duration::from_secs(20))
101 .with_max_times(5)
102}
103
104pub(crate) async fn send_configured(
107 transport: &dyn Transport,
108 call: Call,
109 accept: &[u16],
110 policy: Option<RetryPolicy>,
111 clock: &dyn crate::clock::Clock,
112) -> Result<(Reply, RetryDetails), Error> {
113 send_cancellable(transport, call, accept, policy, clock, None).await
114}
115
116pub(crate) async fn send_cancellable(
117 transport: &dyn Transport,
118 call: Call,
119 accept: &[u16],
120 policy: Option<RetryPolicy>,
121 clock: &dyn crate::clock::Clock,
122 cancellation: Option<&RetryCancellation>,
123) -> Result<(Reply, RetryDetails), Error> {
124 use backon::BackoffBuilder;
125 let started = clock.elapsed();
126 let mut details = RetryDetails::default();
127 let mut delays = retry_delays().without_max_times().build();
128 let mut ordinary_retries = 0;
129 loop {
130 if cancellation.is_some_and(RetryCancellation::is_cancelled) {
131 return Err(Error::new(
132 ErrorKind::Interrupted,
133 "launch retries cancelled before a VM was assigned",
134 )
135 .with_retry_details(details.attempt_count, details.backoff));
136 }
137 details.attempt_count += 1;
138 let error = match transport.send(call.clone()).await {
139 Ok(reply) if reply.is_success() || accept.contains(&reply.status) => {
140 return Ok((reply, details));
141 }
142 Ok(reply) => classify_failure_in_region(call.operation, &reply, transport.region()),
143 Err(error) => error,
144 };
145 let delay = delays.next().expect("unbounded backoff");
146 let elapsed = clock.elapsed().saturating_sub(started);
147 let allowed = if error.throttled()
148 && let Some(policy) = policy
149 {
150 details.attempt_count < policy.max_attempts
151 && elapsed.saturating_add(delay) < policy.max_elapsed
152 } else {
153 ordinary_retries += 1;
154 ordinary_retries <= 5
155 };
156 if !error.retryable() || !allowed {
157 return Err(error.with_retry_details(details.attempt_count, details.backoff));
158 }
159 if let Some(cancellation) = cancellation {
160 tokio::select! {
161 _ = cancellation.cancelled() => continue,
162 _ = clock.sleep(delay) => {}
163 }
164 } else {
165 clock.sleep(delay).await;
166 }
167 details.backoff += delay;
168 if error.throttled()
170 && policy.is_some_and(|p| clock.elapsed().saturating_sub(started) >= p.max_elapsed)
171 {
172 return Err(error.with_retry_details(details.attempt_count, details.backoff));
173 }
174 }
175}
176
177const API_PATH_VERSION: &str = crate::constants::MODEL_API_VERSION;
184
185pub const SIGNING_NAME: &str = "lambda";
191
192#[derive(Clone, Debug, Eq, PartialEq)]
198pub struct Call {
199 pub operation: &'static str,
205 pub method: Method,
208 pub path: String,
211 pub body: Option<Vec<u8>>,
213}
214
215#[derive(Clone, Copy, Debug, Eq, PartialEq)]
229pub enum Method {
230 Get,
231 Post,
232 Patch,
233 Delete,
234}
235
236impl Method {
237 pub fn as_str(self) -> &'static str {
238 match self {
239 Method::Get => "GET",
240 Method::Post => "POST",
241 Method::Patch => "PATCH",
242 Method::Delete => "DELETE",
243 }
244 }
245}
246
247impl Call {
248 pub fn get(operation: &'static str, path: impl Into<String>) -> Self {
250 Self {
251 operation,
252 method: Method::Get,
253 path: path.into(),
254 body: None,
255 }
256 }
257
258 pub fn post_json<T: serde::Serialize>(
260 operation: &'static str,
261 path: impl Into<String>,
262 body: &T,
263 ) -> Result<Self, Error> {
264 let body = serde_json::to_vec(body).map_err(|error| {
265 Error::new(
266 ErrorKind::Unexpected,
267 format!("could not serialize the {operation} request body: {error}"),
268 )
269 })?;
270 Ok(Self {
271 operation,
272 method: Method::Post,
273 path: path.into(),
274 body: Some(body),
275 })
276 }
277
278 pub fn patch_json<T: serde::Serialize>(
286 operation: &'static str,
287 path: impl Into<String>,
288 body: &T,
289 ) -> Result<Self, Error> {
290 let body = serde_json::to_vec(body).map_err(|error| {
291 Error::new(
292 ErrorKind::Unexpected,
293 format!("could not serialize the {operation} request body: {error}"),
294 )
295 })?;
296 Ok(Self {
297 operation,
298 method: Method::Patch,
299 path: path.into(),
300 body: Some(body),
301 })
302 }
303
304 pub fn post_empty(operation: &'static str, path: impl Into<String>) -> Self {
311 Self {
312 operation,
313 method: Method::Post,
314 path: path.into(),
315 body: Some(b"{}".to_vec()),
316 }
317 }
318
319 pub fn delete(operation: &'static str, path: impl Into<String>) -> Self {
321 Self {
322 operation,
323 method: Method::Delete,
324 path: path.into(),
325 body: None,
326 }
327 }
328
329 pub fn body_bytes(&self) -> &[u8] {
331 self.body.as_deref().unwrap_or(&[])
332 }
333}
334
335#[derive(Clone, Debug)]
342pub struct Reply {
343 pub status: u16,
344 pub body: Vec<u8>,
345 pub error_type: Option<String>,
349}
350
351impl Reply {
352 pub fn json<T: serde::de::DeserializeOwned>(&self, operation: &str) -> Result<T, Error> {
359 serde_json::from_slice(&self.body).map_err(|error| {
360 let text = String::from_utf8_lossy(&self.body);
361 let shown: String = text.chars().take(512).collect();
362 Error::new(
363 ErrorKind::Platform,
364 format!(
365 "could not read the {operation} response ({error}). The service model for \
366 {API_PATH_VERSION} is what this client's shapes are transcribed from, so a \
367 member that has moved is a drift-gate failure rather than a transport bug. \
368 Body: {shown}"
369 ),
370 )
371 .with_source(error)
372 })
373 }
374
375 fn is_success(&self) -> bool {
377 (200..300).contains(&self.status)
378 }
379}
380
381pub trait Transport: Send + Sync {
387 fn region(&self) -> Option<&Region> {
390 None
391 }
392
393 fn send(
400 &self,
401 call: Call,
402 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Reply, Error>> + Send + '_>>;
403
404 fn resolve_credentials(
411 &self,
412 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), Error>> + Send + '_>> {
413 Box::pin(async { Ok(()) })
414 }
415}
416
417pub async fn send_with_retry(transport: &dyn Transport, call: Call) -> Result<Reply, Error> {
435 use backon::Retryable;
436
437 let operation = call.operation;
438 let attempt = || {
439 let call = call.clone();
440 async move {
441 let reply = transport.send(call).await?;
442 if reply.is_success() {
443 return Ok(reply);
444 }
445 Err(classify_failure_in_region(
446 operation,
447 &reply,
448 transport.region(),
449 ))
450 }
451 };
452
453 let mut details = RetryDetails {
454 attempt_count: 1,
455 backoff: Duration::ZERO,
456 };
457 let result = attempt
458 .retry(retry_delays())
459 .when(Error::retryable)
460 .notify(|_, delay| {
461 details.attempt_count += 1;
462 details.backoff += delay;
463 })
464 .await;
465 result.map_err(|error| error.with_retry_details(details.attempt_count, details.backoff))
466}
467
468pub async fn send_accepting(
476 transport: &dyn Transport,
477 call: Call,
478 accept: &[u16],
479) -> Result<Reply, Error> {
480 use backon::Retryable;
481
482 let operation = call.operation;
483 let attempt = || {
484 let call = call.clone();
485 async move {
486 let reply = transport.send(call).await?;
487 if reply.is_success() || accept.contains(&reply.status) {
488 return Ok(reply);
489 }
490 Err(classify_failure_in_region(
491 operation,
492 &reply,
493 transport.region(),
494 ))
495 }
496 };
497
498 let mut details = RetryDetails {
499 attempt_count: 1,
500 backoff: Duration::ZERO,
501 };
502 let result = attempt
503 .retry(retry_delays())
504 .when(Error::retryable)
505 .notify(|_, delay| {
506 details.attempt_count += 1;
507 details.backoff += delay;
508 })
509 .await;
510 result.map_err(|error| error.with_retry_details(details.attempt_count, details.backoff))
511}
512
513pub fn classify_failure(operation: &str, reply: &Reply) -> Error {
534 classify_failure_in_region(operation, reply, None)
535}
536
537fn classify_failure_in_region(operation: &str, reply: &Reply, region: Option<&Region>) -> Error {
538 let parsed: crate::control::ops::ServiceErrorWire =
539 serde_json::from_slice(&reply.body).unwrap_or_default();
540 let detail = parsed.message.as_deref().unwrap_or("").trim().to_string();
541
542 let (kind, explanation) = match reply.status {
543 status if reply.error_type.as_deref() == Some("ThrottlingException") => (
548 ErrorKind::Retryable,
549 format!(
550 "ThrottlingException, named by the x-amzn-ErrorType header on an HTTP {status} \
551 — retry the identical request."
552 ),
553 ),
554 400 => (
555 ErrorKind::Platform,
556 "ValidationException — the service refused a value. Every constraint the pinned \
557 service model states on a member this client sends is checked locally before the \
558 call — min as well as max, pattern, and enum, because nothing else validates a \
559 request here: this client signs with aws-sigv4 and sends with reqwest, so botocore's \
560 VALIDATED_METADATA_ATTRS never applies to it. So reaching this status means a \
561 constraint this client does not yet check, and closing it is a guard in \
562 crates/microvms-app/src/control plus a line in tools/check-model-drift.py."
563 .to_string(),
564 ),
565 402 => (
566 ErrorKind::Platform,
567 "ServiceQuotaExceededException — an account limit, not a request defect.".to_string(),
568 ),
569 403 if detail.is_empty() => (
570 ErrorKind::Credentials,
571 null_message_denial(operation, region),
572 ),
573 403 => (
574 ErrorKind::Credentials,
575 "AccessDeniedException, with a message — the service reported a genuine denial."
576 .to_string(),
577 ),
578 404 => (
579 ErrorKind::Platform,
580 "ResourceNotFoundException — the image or MicroVM does not exist.".to_string(),
581 ),
582 409 => (
583 ErrorKind::Platform,
584 "ConflictException — the resource is in a state that forbids this call. An image in \
585 CREATING refuses deletion, which is why teardown retries."
586 .to_string(),
587 ),
588 429 => (
589 ErrorKind::Retryable,
590 "ThrottlingException — retry the identical request.".to_string(),
591 ),
592 status if status >= 500 => (
593 ErrorKind::Retryable,
594 "InternalServerException — a service fault, retry the identical request.".to_string(),
595 ),
596 _ => (
597 ErrorKind::Platform,
598 "an unmodeled status: the service model declares 400, 402, 403, 404, 409, 429, and \
599 500 for these operations."
600 .to_string(),
601 ),
602 };
603
604 let said = if detail.is_empty() {
605 "the service sent no message".to_string()
606 } else {
607 format!("the service said {detail:?}")
608 };
609 Error::new(
610 kind,
611 format!(
612 "{operation} failed with HTTP {}: {said}. {explanation}",
613 reply.status
614 ),
615 )
616 .with_throttled(
617 reply.status == 429 || reply.error_type.as_deref() == Some("ThrottlingException"),
618 )
619}
620
621fn null_message_denial(operation: &str, region: Option<&Region>) -> String {
623 let availability = match region {
624 Some(region) if region.is_measured() => format!(
625 "{} carries MicroVMs; an IAM policy is the likely cause.",
626 region.as_str()
627 ),
628 Some(region) => format!(
629 "{} is not in the known MicroVMs region list; verify service availability as well as the IAM policy.",
630 region.as_str()
631 ),
632 None => "The cause may be an unsupported region or an IAM denial.".to_string(),
633 };
634 let region_arg = region
635 .map(|region| format!(" --region {}", region.as_str()))
636 .unwrap_or_default();
637 format!(
638 "AccessDeniedException with an empty or null message field. {availability} \
639 A create call with tags can require the implied lambda:TagResource action. \
640 Check the full denial in CloudTrail: aws cloudtrail lookup-events{region_arg} \
641 --lookup-attributes AttributeKey=EventName,AttributeValue={operation} \
642 --query 'Events[].CloudTrailEvent' --output text. Inspect errorCode and errorMessage \
643 for the denied action (docs/PLATFORM.md)."
644 )
645}
646
647pub fn error_type_name(header: &str) -> &str {
654 let name = header.split(':').next().unwrap_or(header);
655 name.split_once('#').map_or(name, |(_, after)| after).trim()
656}
657
658pub fn endpoint_for(region: &Region) -> String {
663 format!("https://{SIGNING_NAME}.{}.amazonaws.com", region.as_str())
664}
665
666pub mod paths {
673 use super::API_PATH_VERSION;
674
675 pub fn encode_segment(segment: &str) -> String {
683 use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
684 const ESCAPED: &AsciiSet = &NON_ALPHANUMERIC
685 .remove(b'-')
686 .remove(b'_')
687 .remove(b'.')
688 .remove(b'~');
689 utf8_percent_encode(segment, ESCAPED).to_string()
690 }
691
692 pub fn microvm_images() -> String {
694 format!("/{API_PATH_VERSION}/microvm-images")
695 }
696
697 pub fn microvm_images_list(name_filter: Option<&str>, next_token: Option<&str>) -> String {
706 let mut query: Vec<String> = Vec::new();
707 if let Some(filter) = name_filter {
708 query.push(format!("nameFilter={}", encode_segment(filter)));
709 }
710 if let Some(token) = next_token {
711 query.push(format!("nextToken={}", encode_segment(token)));
712 }
713 if query.is_empty() {
714 return microvm_images();
715 }
716 format!("{}?{}", microvm_images(), query.join("&"))
717 }
718
719 pub fn microvm_image(image: &str) -> String {
721 format!(
722 "/{API_PATH_VERSION}/microvm-images/{}",
723 encode_segment(image)
724 )
725 }
726
727 fn with_next_token(path: String, next_token: Option<&str>) -> String {
736 match next_token {
737 Some(token) => format!("{path}?nextToken={}", encode_segment(token)),
738 None => path,
739 }
740 }
741
742 pub fn image_versions(image: &str, next_token: Option<&str>) -> String {
750 image_versions_paged(image, next_token, None)
751 }
752
753 pub fn image_versions_paged(
782 image: &str,
783 next_token: Option<&str>,
784 max_results: Option<u32>,
785 ) -> String {
786 let mut query: Vec<String> = Vec::new();
787 if let Some(size) = max_results {
788 query.push(format!("maxResults={}", size.clamp(1, 50)));
789 }
790 if let Some(token) = next_token {
791 query.push(format!("nextToken={}", encode_segment(token)));
792 }
793 let path = format!("{}/versions", microvm_image(image));
794 if query.is_empty() {
795 return path;
796 }
797 format!("{path}?{}", query.join("&"))
798 }
799
800 pub fn image_version(image: &str, version: &str) -> String {
811 format!(
812 "{}/versions/{}",
813 microvm_image(image),
814 encode_segment(version)
815 )
816 }
817
818 pub fn image_build(image: &str, version: &str, build_id: &str) -> String {
824 format!(
825 "{}/builds/{}",
826 image_version(image, version),
827 encode_segment(build_id)
828 )
829 }
830
831 pub fn managed_microvm_images(next_token: Option<&str>) -> String {
838 with_next_token(
839 format!("/{API_PATH_VERSION}/managed-microvm-images"),
840 next_token,
841 )
842 }
843
844 pub fn managed_image_versions(image: &str, next_token: Option<&str>) -> String {
852 with_next_token(
853 format!(
854 "/{API_PATH_VERSION}/managed-microvm-images/{}/versions",
855 encode_segment(image)
856 ),
857 next_token,
858 )
859 }
860
861 pub fn image_builds(image: &str, version: &str, next_token: Option<&str>) -> String {
864 with_next_token(
865 format!("{}/builds", image_version(image, version)),
866 next_token,
867 )
868 }
869
870 pub fn microvms() -> String {
877 format!("/{API_PATH_VERSION}/microvms")
878 }
879
880 pub fn microvms_list(next_token: Option<&str>) -> String {
882 with_next_token(microvms(), next_token)
883 }
884
885 pub fn microvms_list_matching(
889 image_identifier: Option<&str>,
890 image_version: Option<&str>,
891 next_token: Option<&str>,
892 ) -> String {
893 let query: Vec<String> = [
894 ("imageIdentifier", image_identifier),
895 ("imageVersion", image_version),
896 ("nextToken", next_token),
897 ]
898 .into_iter()
899 .filter_map(|(key, value)| value.map(|value| format!("{key}={}", encode_segment(value))))
900 .collect();
901 if query.is_empty() {
902 return microvms();
903 }
904 format!("{}?{}", microvms(), query.join("&"))
905 }
906
907 pub fn microvm(id: &str) -> String {
909 format!("/{API_PATH_VERSION}/microvms/{}", encode_segment(id))
910 }
911
912 pub fn suspend(id: &str) -> String {
914 format!("{}/suspend", microvm(id))
915 }
916
917 pub fn resume(id: &str) -> String {
919 format!("{}/resume", microvm(id))
920 }
921
922 pub fn auth_token(id: &str) -> String {
924 format!("{}/auth-token", microvm(id))
925 }
926
927 pub fn shell_auth_token(id: &str) -> String {
934 format!("{}/shell-auth-token", microvm(id))
935 }
936}
937
938#[cfg(test)]
939mod tests {
940 use super::*;
941
942 #[tokio::test]
943 async fn throttle_budget_counts_attempts_bounds_elapsed_and_keeps_other_failures_default() {
944 use crate::clock::Clock;
945 use crate::control::fake::{Answer, FakeControlPlane, TestClock};
946 for (status, policy, expected) in [
947 (429, None, 6),
948 (
949 429,
950 Some(RetryPolicy::new(3, Duration::from_secs(3600)).unwrap()),
951 3,
952 ),
953 (
954 429,
955 Some(RetryPolicy::new(9, Duration::from_millis(1)).unwrap()),
956 1,
957 ),
958 (
959 503,
960 Some(RetryPolicy::new(9, Duration::from_secs(3600)).unwrap()),
961 6,
962 ),
963 (
964 403,
965 Some(RetryPolicy::new(9, Duration::from_secs(3600)).unwrap()),
966 1,
967 ),
968 ] {
969 let transport = FakeControlPlane::new();
970 for _ in 0..expected {
971 transport.answer("RunMicrovm", Answer::failure(status, "failure"));
972 }
973 transport.answer("RunMicrovm", Answer::ok("{}"));
976 let clock = TestClock::new();
977 let error = send_configured(
978 &transport,
979 Call::get("RunMicrovm", "/run"),
980 &[],
981 policy,
982 &clock,
983 )
984 .await
985 .unwrap_err();
986 assert_eq!(transport.call_count("RunMicrovm"), expected);
987 assert_eq!(error.attempt_count(), expected as u32);
988 assert_eq!(error.throttled(), status == 429);
989 assert_eq!(error.backoff(), clock.elapsed());
990 if expected == 1 {
991 assert_eq!(clock.elapsed(), Duration::ZERO);
992 }
993 }
994 assert!(RetryPolicy::new(0, Duration::from_secs(1)).is_err());
995 assert!(RetryPolicy::new(1, Duration::ZERO).is_err());
996 }
997
998 #[tokio::test]
999 async fn a_retry_at_the_elapsed_budget_boundary_does_not_sleep() {
1000 use crate::clock::Clock;
1001 use crate::control::fake::{Answer, FakeControlPlane, TestClock};
1002 let transport = FakeControlPlane::new();
1003 transport.answer("RunMicrovm", Answer::throttled_as_400());
1004 let gate = transport.hold_responses();
1005 let clock = TestClock::new();
1006 clock.set_unix_now(Duration::ZERO);
1007 let pending = send_configured(
1008 &transport,
1009 Call::get("RunMicrovm", "/run"),
1010 &[],
1011 Some(RetryPolicy::new(2, Duration::MAX).unwrap()),
1012 &clock,
1013 );
1014 tokio::pin!(pending);
1015 tokio::select! {
1016 biased;
1017 _ = &mut pending => panic!("response must wait for the gate"),
1018 _ = std::future::ready(()) => {}
1019 }
1020 assert_eq!(transport.call_count("RunMicrovm"), 1);
1021 clock.advance(Duration::MAX);
1024 gate.notify_one();
1025 let error = tokio::select! {
1028 biased;
1029 result = &mut pending => result.unwrap_err(),
1030 _ = std::future::ready(()) => panic!("elapsed budget must stop before another request"),
1031 };
1032 assert!(error.throttled());
1033 assert_eq!(error.attempt_count(), 1);
1034 assert_eq!(transport.call_count("RunMicrovm"), 1);
1035 assert_eq!(error.backoff(), Duration::ZERO);
1036 assert_eq!(clock.elapsed(), Duration::MAX);
1037 }
1038
1039 #[tokio::test]
1040 async fn configured_retries_recover_header_throttles_with_the_identical_call() {
1041 use crate::clock::Clock;
1042 use crate::control::fake::{Answer, FakeControlPlane, TestClock};
1043 let transport = FakeControlPlane::new();
1044 for _ in 0..7 {
1045 transport.answer("RunMicrovm", Answer::throttled_as_400());
1046 }
1047 transport.answer("RunMicrovm", Answer::ok("{}"));
1048 let clock = TestClock::new();
1049 let call = Call::post_json(
1050 "RunMicrovm",
1051 "/run",
1052 &serde_json::json!({"clientToken":"same-launch"}),
1053 )
1054 .unwrap();
1055 let (_, details) = send_configured(
1056 &transport,
1057 call.clone(),
1058 &[],
1059 Some(RetryPolicy::new(8, Duration::from_secs(3600)).unwrap()),
1060 &clock,
1061 )
1062 .await
1063 .unwrap();
1064 assert_eq!(details.attempt_count, 8);
1065 assert_eq!(details.backoff, clock.elapsed());
1066 assert!(details.backoff > Duration::ZERO);
1067 assert!(transport.calls().iter().all(|sent| sent == &call));
1068 transport.answer("GetMicrovm", Answer::failure(404, "missing"));
1069 let (reply, details) = send_configured(
1070 &transport,
1071 Call::get("GetMicrovm", "/missing"),
1072 &[404],
1073 None,
1074 &clock,
1075 )
1076 .await
1077 .unwrap();
1078 assert_eq!(reply.status, 404);
1079 assert_eq!(details.attempt_count, 1);
1080 assert_eq!(details.backoff, Duration::ZERO);
1081 }
1082
1083 #[tokio::test(start_paused = true)]
1084 async fn exhausted_public_retry_reports_throttle_attempts_and_backoff() {
1085 use crate::control::fake::{Answer, FakeControlPlane};
1086 for accepts_missing in [false, true] {
1087 let transport = FakeControlPlane::new();
1088 for _ in 0..6 {
1089 transport.answer("ListMicrovms", Answer::throttled_as_400());
1090 }
1091 transport.answer("ListMicrovms", Answer::ok("{}"));
1092 let call = Call::get("ListMicrovms", "/list");
1093 let error = if accepts_missing {
1094 send_accepting(&transport, call, &[404]).await
1095 } else {
1096 send_with_retry(&transport, call).await
1097 }
1098 .unwrap_err();
1099 assert_eq!(transport.call_count("ListMicrovms"), 6);
1100 assert!(error.throttled());
1101 assert_eq!(error.attempt_count(), 6);
1102 assert!(error.backoff() >= Duration::from_millis(6200));
1103 assert!(error.backoff() < Duration::from_millis(12400));
1104 }
1105 for status in [200, 404] {
1106 let transport = FakeControlPlane::new();
1107 transport.answer("GetMicrovm", Answer::failure(status, "accepted"));
1108 let reply = send_accepting(&transport, Call::get("GetMicrovm", "/missing"), &[404])
1109 .await
1110 .unwrap();
1111 assert_eq!(reply.status, status);
1112 assert_eq!(transport.call_count("GetMicrovm"), 1);
1113 }
1114 }
1115
1116 #[tokio::test]
1117 async fn a_cancel_signal_wakes_backoff_before_another_request() {
1118 use crate::clock::testing::YieldingClock;
1119 use crate::control::fake::{Answer, FakeControlPlane};
1120 let transport = FakeControlPlane::new();
1121 transport.answer("RunMicrovm", Answer::throttled_as_400());
1122 let clock = YieldingClock::new();
1123 let cancellation = RetryCancellation::default();
1124 let call = Call::get("RunMicrovm", "/run");
1125 let pending = send_cancellable(
1126 &transport,
1127 call,
1128 &[],
1129 Some(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap()),
1130 &clock,
1131 Some(&cancellation),
1132 );
1133 tokio::pin!(pending);
1134 tokio::select! { biased; _ = &mut pending => panic!("backoff should yield"), _ = std::future::ready(()) => {} }
1135 cancellation.cancel();
1136 let error = pending.await.unwrap_err();
1137 assert_eq!(error.kind(), ErrorKind::Interrupted);
1138 assert_eq!(transport.call_count("RunMicrovm"), 1);
1139 }
1140
1141 #[tokio::test]
1142 async fn dropping_a_retry_future_prevents_another_request() {
1143 use crate::clock::testing::YieldingClock;
1144 use crate::control::fake::{Answer, FakeControlPlane};
1145 let transport = FakeControlPlane::new();
1146 transport.answer("RunMicrovm", Answer::throttled_as_400());
1147 let clock = YieldingClock::new();
1148 let mut future = Box::pin(send_configured(
1149 &transport,
1150 Call::get("RunMicrovm", "/run"),
1151 &[],
1152 Some(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap()),
1153 &clock,
1154 ));
1155 tokio::select! { biased; _ = &mut future => panic!("retry sleep should yield"), _ = std::future::ready(()) => {} }
1156 drop(future);
1157 clock.advance(Duration::from_secs(3600));
1159 tokio::task::yield_now().await;
1160 tokio::task::yield_now().await;
1161 assert_eq!(transport.call_count("RunMicrovm"), 1);
1162 }
1163
1164 #[test]
1168 fn the_endpoint_uses_the_models_lambda_prefix_for_every_region() {
1169 assert_eq!(
1170 endpoint_for(&Region::US_EAST_1),
1171 "https://lambda.us-east-1.amazonaws.com"
1172 );
1173 assert_eq!(
1174 endpoint_for(&Region::AP_NORTHEAST_1),
1175 "https://lambda.ap-northeast-1.amazonaws.com"
1176 );
1177 assert_eq!(
1180 endpoint_for(&"us-west-1".parse().expect("a region name")),
1181 "https://lambda.us-west-1.amazonaws.com"
1182 );
1183 }
1184
1185 #[test]
1189 fn every_path_matches_the_models_request_uri() {
1190 assert_eq!(paths::microvm_images(), "/2025-09-09/microvm-images");
1191 assert_eq!(
1192 paths::microvm_image("img"),
1193 "/2025-09-09/microvm-images/img"
1194 );
1195 assert_eq!(
1196 paths::image_versions("img", None),
1197 "/2025-09-09/microvm-images/img/versions"
1198 );
1199 assert_eq!(
1200 paths::image_version("img", "1"),
1201 "/2025-09-09/microvm-images/img/versions/1"
1202 );
1203 assert_eq!(
1204 paths::image_builds("img", "1", None),
1205 "/2025-09-09/microvm-images/img/versions/1/builds"
1206 );
1207 assert_eq!(
1208 paths::image_build("img", "1", "build-abc"),
1209 "/2025-09-09/microvm-images/img/versions/1/builds/build-abc"
1210 );
1211 assert_eq!(
1212 paths::managed_microvm_images(None),
1213 "/2025-09-09/managed-microvm-images"
1214 );
1215 assert_eq!(
1216 paths::managed_image_versions(
1217 "arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1",
1218 None
1219 ),
1220 "/2025-09-09/managed-microvm-images/\
1221 arn%3Aaws%3Alambda%3Aus-east-1%3Aaws%3Amicrovm-image%3Aal2023-1/versions"
1222 );
1223 assert_eq!(paths::microvms(), "/2025-09-09/microvms");
1224 assert_eq!(paths::microvms_list(None), "/2025-09-09/microvms");
1225 assert_eq!(paths::microvm("mvm-1"), "/2025-09-09/microvms/mvm-1");
1226 assert_eq!(
1227 paths::suspend("mvm-1"),
1228 "/2025-09-09/microvms/mvm-1/suspend"
1229 );
1230 assert_eq!(paths::resume("mvm-1"), "/2025-09-09/microvms/mvm-1/resume");
1231 assert_eq!(
1232 paths::auth_token("mvm-1"),
1233 "/2025-09-09/microvms/mvm-1/auth-token"
1234 );
1235 assert_eq!(
1236 paths::shell_auth_token("mvm-1"),
1237 "/2025-09-09/microvms/mvm-1/shell-auth-token"
1238 );
1239 }
1240
1241 #[test]
1248 fn the_image_listing_path_encodes_its_query_members() {
1249 assert_eq!(
1250 paths::microvm_images_list(None, None),
1251 "/2025-09-09/microvm-images"
1252 );
1253 assert_eq!(
1254 paths::microvm_images_list(Some("my-image"), None),
1255 "/2025-09-09/microvm-images?nameFilter=my-image"
1256 );
1257 assert_eq!(
1258 paths::microvm_images_list(Some("my-image"), Some("a+b/c=")),
1259 "/2025-09-09/microvm-images?nameFilter=my-image&nextToken=a%2Bb%2Fc%3D"
1260 );
1261 assert_eq!(
1262 paths::microvm_images_list(None, Some("token")),
1263 "/2025-09-09/microvm-images?nextToken=token"
1264 );
1265 }
1266
1267 #[test]
1278 fn the_other_paginated_listings_encode_their_cursor_too() {
1279 assert_eq!(
1280 paths::image_versions("img", Some("a+b/c=")),
1281 "/2025-09-09/microvm-images/img/versions?nextToken=a%2Bb%2Fc%3D"
1282 );
1283 assert_eq!(
1284 paths::image_builds("img", "1", Some("a+b/c=")),
1285 "/2025-09-09/microvm-images/img/versions/1/builds?nextToken=a%2Bb%2Fc%3D"
1286 );
1287 assert_eq!(
1288 paths::microvms_list(Some("a+b/c=")),
1289 "/2025-09-09/microvms?nextToken=a%2Bb%2Fc%3D"
1290 );
1291
1292 for path in [
1295 paths::image_versions("img", None),
1296 paths::image_builds("img", "1", None),
1297 paths::microvms_list(None),
1298 ] {
1299 assert!(
1300 !path.contains('?'),
1301 "an absent cursor emits no query: {path}"
1302 );
1303 }
1304 }
1305
1306 #[test]
1317 fn the_version_listings_explicit_page_size_is_ordered_and_clamped() {
1318 assert_eq!(
1319 paths::image_versions_paged("img", None, Some(1)),
1320 "/2025-09-09/microvm-images/img/versions?maxResults=1"
1321 );
1322 assert_eq!(
1325 paths::image_versions_paged("img", Some("a+b/c="), Some(2)),
1326 "/2025-09-09/microvm-images/img/versions?maxResults=2&nextToken=a%2Bb%2Fc%3D"
1327 );
1328 assert_eq!(
1329 paths::image_versions_paged("img", None, Some(0)),
1330 "/2025-09-09/microvm-images/img/versions?maxResults=1",
1331 "the model's min is 1; a 0 would be refused"
1332 );
1333 assert_eq!(
1334 paths::image_versions_paged("img", None, Some(999)),
1335 "/2025-09-09/microvm-images/img/versions?maxResults=50",
1336 "the model's max is 50; a larger value would be refused"
1337 );
1338
1339 assert_eq!(
1342 paths::image_versions_paged("img", None, None),
1343 paths::image_versions("img", None)
1344 );
1345 assert!(
1346 !paths::image_versions("img", Some("t")).contains("maxResults"),
1347 "no production path names a page size: {}",
1348 paths::image_versions("img", Some("t"))
1349 );
1350 }
1351
1352 #[test]
1365 fn the_managed_routes_are_their_own_collection_and_encode_their_cursor() {
1366 assert!(
1367 !paths::managed_microvm_images(None).contains('?'),
1368 "an absent cursor emits no query member"
1369 );
1370 assert_eq!(
1371 paths::managed_microvm_images(Some("a+b/c=")),
1372 "/2025-09-09/managed-microvm-images?nextToken=a%2Bb%2Fc%3D"
1373 );
1374 assert_eq!(
1375 paths::managed_image_versions("al2023-1", Some("a+b/c=")),
1376 "/2025-09-09/managed-microvm-images/al2023-1/versions?nextToken=a%2Bb%2Fc%3D"
1377 );
1378
1379 assert_ne!(
1382 paths::managed_microvm_images(None),
1383 paths::microvm_images_list(None, None)
1384 );
1385 assert!(
1386 !paths::microvm_images().contains("managed"),
1387 "the account's own collection must not acquire the managed prefix"
1388 );
1389 }
1390
1391 #[test]
1398 fn one_builds_path_is_the_listing_plus_an_encoded_segment() {
1399 let path = paths::image_build("img", "1", "4a4c5e30-811f-47fa-9893-260ea6a37a8f");
1400 assert_eq!(
1401 path,
1402 "/2025-09-09/microvm-images/img/versions/1/builds/4a4c5e30-811f-47fa-9893-260ea6a37a8f"
1403 );
1404 assert!(!path.contains('?'), "{path}");
1405 assert!(
1406 path.starts_with(&paths::image_builds("img", "1", None)),
1407 "the get path must extend the list path rather than be a second route: {path}"
1408 );
1409 assert_eq!(
1411 paths::image_build("img", "1", "a/b"),
1412 "/2025-09-09/microvm-images/img/versions/1/builds/a%2Fb"
1413 );
1414 }
1415
1416 #[test]
1420 fn deleting_one_version_carries_no_pagination_cursor() {
1421 let path = paths::image_version("img", "1");
1422 assert_eq!(path, "/2025-09-09/microvm-images/img/versions/1");
1423 assert!(!path.contains('?'), "{path}");
1424 assert!(!path.contains("nextToken"), "{path}");
1425 }
1426
1427 #[test]
1438 fn a_real_arn_identifier_is_percent_encoded_colons_and_all() {
1439 let arn = "arn:aws:lambda:us-east-1:123456789012:microvm-image:img";
1440 let path = paths::microvm_image(arn);
1441 assert_eq!(
1442 path,
1443 "/2025-09-09/microvm-images/arn%3Aaws%3Alambda%3Aus-east-1%3A123456789012%3Amicrovm-image%3Aimg"
1444 );
1445 assert!(
1446 !path["/2025-09-09/microvm-images/".len()..].contains(':'),
1447 "an unencoded colon in a path segment is a signature mismatch: {path}"
1448 );
1449 }
1450
1451 #[test]
1460 fn the_segment_encoder_encodes_slashes_wherever_they_come_from() {
1461 assert_eq!(paths::encode_segment("a/b"), "a%2Fb");
1462 let path = paths::microvm_image("has/a/slash");
1463 assert_eq!(path, "/2025-09-09/microvm-images/has%2Fa%2Fslash");
1464 assert!(
1465 !path["/2025-09-09/microvm-images/".len()..].contains('/'),
1466 "an encoded identifier contributes no path separators: {path}"
1467 );
1468 }
1469
1470 #[test]
1474 fn the_path_version_is_the_pinned_model_version() {
1475 assert_eq!(API_PATH_VERSION, crate::constants::MODEL_API_VERSION);
1476 assert!(paths::microvms().starts_with("/2025-09-09/"));
1477 }
1478
1479 #[test]
1482 fn the_signing_name_is_lambda_not_lambda_microvms() {
1483 assert_eq!(SIGNING_NAME, "lambda");
1484 }
1485
1486 #[test]
1488 fn a_null_message_access_denied_names_both_possible_causes() {
1489 let reply = Reply {
1490 status: 403,
1491 body: br#"{"message": null}"#.to_vec(),
1492 error_type: None,
1493 };
1494 let error = classify_failure("ListMicrovms", &reply);
1495 assert_eq!(error.kind(), ErrorKind::Credentials);
1496 let message = error.to_string();
1497 assert!(message.contains("null"), "{message}");
1498 assert!(message.contains("region"), "{message}");
1499 assert!(
1500 message.contains("unsupported region or an IAM denial"),
1501 "{message}"
1502 );
1503 assert!(message.contains("lambda:TagResource"), "{message}");
1504 assert!(message.contains("cloudtrail lookup-events"), "{message}");
1505 }
1506
1507 #[tokio::test]
1509 async fn supported_region_null_denials_point_to_iam_in_both_send_paths() {
1510 use crate::control::fake::{Answer, FakeControlPlane};
1511 let transport = FakeControlPlane::in_region(Region::US_EAST_1);
1512 transport.answer(
1513 "CreateMicrovmImage",
1514 Answer {
1515 status: 403,
1516 body: r#"{"message":null}"#.to_string(),
1517 error_type: Some("AccessDeniedException".to_string()),
1518 },
1519 );
1520 for accept in [false, true] {
1521 let call = Call::get("CreateMicrovmImage", "/images");
1522 let error = if accept {
1523 send_accepting(&transport, call, &[404]).await.unwrap_err()
1524 } else {
1525 send_with_retry(&transport, call).await.unwrap_err()
1526 };
1527 assert_eq!(error.kind(), ErrorKind::Credentials);
1528 assert!(!error.retryable());
1529 let message = error.to_string();
1530 for detail in [
1531 "us-east-1 carries MicroVMs",
1532 "policy is the likely cause",
1533 "lambda:TagResource",
1534 "cloudtrail lookup-events --region us-east-1",
1535 "AttributeValue=CreateMicrovmImage",
1536 "errorMessage",
1537 ] {
1538 assert!(message.contains(detail), "{message}");
1539 }
1540 assert!(!message.contains("Check the region before the policy"));
1541 }
1542 assert_eq!(transport.call_count("CreateMicrovmImage"), 2);
1543 }
1544
1545 #[test]
1548 fn unlisted_region_null_denials_keep_both_causes_visible() {
1549 let reply = Reply {
1550 status: 403,
1551 body: b"{}".to_vec(),
1552 error_type: None,
1553 };
1554 let error = classify_failure_in_region(
1555 "ListMicrovms",
1556 &reply,
1557 Some(&"us-west-1".parse().expect("a region name")),
1558 );
1559 let message = error.to_string();
1560 assert!(message.contains("verify service availability as well as the IAM policy"));
1561 assert!(message.contains("--region us-west-1"));
1562 assert!(!message.contains("policy is the likely cause"));
1563 }
1564
1565 #[test]
1569 fn a_403_with_a_message_is_reported_as_a_genuine_denial() {
1570 let reply = Reply {
1571 status: 403,
1572 body: br#"{"message": "User: arn:aws:iam::1:user/u is not authorized"}"#.to_vec(),
1573 error_type: None,
1574 };
1575 let error = classify_failure("RunMicrovm", &reply);
1576 assert_eq!(error.kind(), ErrorKind::Credentials);
1577 let message = error.to_string();
1578 assert!(message.contains("not authorized"), "{message}");
1579 assert!(
1580 message.contains("genuine denial"),
1581 "must distinguish itself from the null-message case: {message}"
1582 );
1583 assert!(
1584 !message.contains("does not carry MicroVMs"),
1585 "must not blame the region when the service named a principal: {message}"
1586 );
1587 }
1588
1589 #[test]
1593 fn a_400_names_the_validation_refusal_and_the_local_checks_it_passed() {
1594 let reply = Reply {
1595 status: 400,
1596 body: br#"{"message": "1 validation error detected"}"#.to_vec(),
1597 error_type: None,
1598 };
1599 let error = classify_failure("RunMicrovm", &reply);
1600 assert_eq!(error.kind(), ErrorKind::Platform);
1601 let message = error.to_string();
1602 assert!(message.contains("ValidationException"), "{message}");
1603 assert!(message.contains("checked locally"), "{message}");
1604 assert!(!message.contains("unmodeled status"), "{message}");
1605 }
1606
1607 #[test]
1617 fn a_throttle_named_by_the_error_type_header_is_retryable_whatever_the_status() {
1618 let reply = Reply {
1619 status: 400,
1620 body: br#"{"message":null,"throttlingReasons":null}"#.to_vec(),
1621 error_type: Some("ThrottlingException".to_string()),
1622 };
1623 let error = classify_failure("ListMicrovms", &reply);
1624 assert!(error.retryable(), "{error}");
1625 assert!(error.throttled(), "the header identifies the throttle");
1626 assert_eq!(error.kind(), ErrorKind::Retryable);
1627 let message = error.to_string();
1628 assert!(message.contains("ThrottlingException"), "{message}");
1629 assert!(message.contains("HTTP 400"), "{message}");
1630 assert!(!message.contains("ValidationException"), "{message}");
1631 }
1632
1633 #[test]
1636 fn a_400_naming_another_error_type_is_still_a_validation_refusal() {
1637 let reply = Reply {
1638 status: 400,
1639 body: br#"{"message": "1 validation error detected"}"#.to_vec(),
1640 error_type: Some("ValidationException".to_string()),
1641 };
1642 let error = classify_failure("RunMicrovm", &reply);
1643 assert!(!error.retryable(), "{error}");
1644 assert!(error.to_string().contains("checked locally"), "{error}");
1645 }
1646
1647 #[test]
1654 fn the_error_type_name_drops_the_uri_suffix_and_the_namespace() {
1655 let cases = [
1656 (
1657 "ThrottlingException:http://internal.amazon.com/coral/com.amazon.coral.availability/",
1658 "ThrottlingException",
1659 ),
1660 (
1661 "aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
1662 "FooError",
1663 ),
1664 (
1665 "com.amazonaws.lambda#ResourceNotFoundException",
1666 "ResourceNotFoundException",
1667 ),
1668 ("ConflictException", "ConflictException"),
1669 ];
1670 for (header, name) in cases {
1671 assert_eq!(error_type_name(header), name, "{header}");
1672 }
1673 }
1674
1675 #[test]
1679 fn only_throttles_and_server_faults_are_retryable() {
1680 let cases = [
1681 (400, false),
1682 (402, false),
1683 (403, false),
1684 (404, false),
1685 (409, false),
1686 (429, true),
1687 (500, true),
1688 (502, true),
1689 (503, true),
1690 ];
1691 for (status, retryable) in cases {
1692 let reply = Reply {
1693 status,
1694 body: br#"{"message": "detail"}"#.to_vec(),
1695 error_type: None,
1696 };
1697 let error = classify_failure("GetMicrovm", &reply);
1698 assert_eq!(error.retryable(), retryable, "HTTP {status}");
1699 assert_eq!(error.throttled(), status == 429, "HTTP {status}");
1700 }
1701 }
1702
1703 #[test]
1707 fn a_control_plane_failure_carries_no_daemon_wire_kind() {
1708 for status in [400, 403, 404, 409, 429, 500] {
1709 let reply = Reply {
1710 status,
1711 body: Vec::new(),
1712 error_type: None,
1713 };
1714 assert_eq!(
1715 classify_failure("GetMicrovm", &reply).wire_kind(),
1716 None,
1717 "HTTP {status} never reached the daemon"
1718 );
1719 }
1720 }
1721
1722 #[test]
1726 fn an_unparseable_error_body_still_classifies_by_status() {
1727 let reply = Reply {
1728 status: 429,
1729 body: b"<html>Too Many Requests</html>".to_vec(),
1730 error_type: None,
1731 };
1732 let error = classify_failure("RunMicrovm", &reply);
1733 assert!(error.retryable(), "a throttle is a throttle");
1734 assert!(error.to_string().contains("no message"), "{error}");
1735 }
1736
1737 #[test]
1741 fn a_response_that_does_not_parse_reports_the_body() {
1742 let reply = Reply {
1743 status: 200,
1744 body: br#"{"microvmId": 42}"#.to_vec(),
1745 error_type: None,
1746 };
1747 let error = reply
1748 .json::<crate::control::ops::MicrovmResponseWire>("GetMicrovm")
1749 .expect_err("microvmId is a string shape");
1750 let message = error.to_string();
1751 assert!(message.contains("GetMicrovm"), "{message}");
1752 assert!(message.contains("microvmId"), "{message}");
1753 assert_eq!(error.kind(), ErrorKind::Platform);
1754 }
1755
1756 #[test]
1760 fn a_bodyless_post_sends_an_empty_json_object() {
1761 let call = Call::post_empty("SuspendMicrovm", paths::suspend("mvm-1"));
1762 assert_eq!(call.body.as_deref(), Some(&b"{}"[..]));
1763 assert_eq!(call.method, Method::Post);
1764 }
1765
1766 #[test]
1769 fn a_get_and_a_delete_carry_no_body() {
1770 let get = Call::get("GetMicrovm", paths::microvm("mvm-1"));
1771 assert_eq!(get.body, None);
1772 assert_eq!(get.body_bytes(), b"");
1773
1774 let delete = Call::delete("TerminateMicrovm", paths::microvm("mvm-1"));
1775 assert_eq!(delete.body, None);
1776 assert_eq!(delete.method, Method::Delete);
1777 }
1778
1779 #[test]
1781 fn the_methods_are_the_ones_the_model_uses_spelled_uppercase() {
1782 assert_eq!(Method::Get.as_str(), "GET");
1783 assert_eq!(Method::Post.as_str(), "POST");
1784 assert_eq!(Method::Patch.as_str(), "PATCH");
1785 assert_eq!(Method::Delete.as_str(), "DELETE");
1786 }
1787
1788 #[test]
1800 fn a_patch_carries_its_body_and_is_not_a_post() {
1801 let call = Call::patch_json(
1802 "UpdateMicrovmImageVersion",
1803 paths::image_version("img", "2.0"),
1804 &crate::control::ops::UpdateImageVersionWire {
1805 status: crate::control::ops::VersionStatus::Inactive,
1806 },
1807 )
1808 .expect("serialises");
1809
1810 assert_eq!(call.method, Method::Patch);
1811 assert_ne!(
1812 call.method,
1813 Method::Post,
1814 "a POST to the version path is a route the model does not declare"
1815 );
1816 assert_eq!(call.path, "/2025-09-09/microvm-images/img/versions/2.0");
1817 assert_eq!(
1818 call.body.as_deref(),
1819 Some(&br#"{"status":"INACTIVE"}"#[..]),
1820 "the one member the request has must reach the body"
1821 );
1822 assert_eq!(call.operation, "UpdateMicrovmImageVersion");
1823 }
1824}