use crate::error::{Error, ErrorKind};
use crate::region::Region;
use std::time::Duration;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RetryPolicy {
max_attempts: u32,
max_elapsed: Duration,
}
impl RetryPolicy {
pub fn new(max_attempts: u32, max_elapsed: Duration) -> Result<Self, Error> {
if max_attempts == 0 || max_elapsed.is_zero() {
return Err(Error::invalid_arg(
"retry max_attempts and max_elapsed must be positive",
));
}
Ok(Self {
max_attempts,
max_elapsed,
})
}
}
#[derive(Clone, Debug)]
pub struct RetryCancellation(tokio::sync::watch::Sender<bool>);
impl Default for RetryCancellation {
fn default() -> Self {
Self(tokio::sync::watch::Sender::new(false))
}
}
impl RetryCancellation {
pub fn cancel(&self) {
self.0.send_replace(true);
}
fn is_cancelled(&self) -> bool {
*self.0.borrow()
}
pub(crate) async fn cancelled(&self) {
let mut receiver = self.0.subscribe();
let _ = receiver.wait_for(|cancelled| *cancelled).await;
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct RetryDetails {
pub attempt_count: u32,
pub backoff: Duration,
}
fn retry_delays() -> backon::ExponentialBuilder {
backon::ExponentialBuilder::default()
.with_jitter()
.with_min_delay(Duration::from_millis(200))
.with_max_delay(Duration::from_secs(20))
.with_max_times(5)
}
pub(crate) async fn send_configured(
transport: &dyn Transport,
call: Call,
accept: &[u16],
policy: Option<RetryPolicy>,
clock: &dyn crate::clock::Clock,
) -> Result<(Reply, RetryDetails), Error> {
send_cancellable(transport, call, accept, policy, clock, None).await
}
pub(crate) async fn send_cancellable(
transport: &dyn Transport,
call: Call,
accept: &[u16],
policy: Option<RetryPolicy>,
clock: &dyn crate::clock::Clock,
cancellation: Option<&RetryCancellation>,
) -> Result<(Reply, RetryDetails), Error> {
use backon::BackoffBuilder;
let started = clock.elapsed();
let mut details = RetryDetails::default();
let mut delays = retry_delays().without_max_times().build();
let mut ordinary_retries = 0;
loop {
if cancellation.is_some_and(RetryCancellation::is_cancelled) {
return Err(Error::new(
ErrorKind::Interrupted,
"launch retries cancelled before a VM was assigned",
)
.with_retry_details(details.attempt_count, details.backoff));
}
details.attempt_count += 1;
let error = match transport.send(call.clone()).await {
Ok(reply) if reply.is_success() || accept.contains(&reply.status) => {
return Ok((reply, details));
}
Ok(reply) => classify_failure_in_region(call.operation, &reply, transport.region()),
Err(error) => error,
};
let delay = delays.next().expect("unbounded backoff");
let elapsed = clock.elapsed().saturating_sub(started);
let allowed = if error.throttled()
&& let Some(policy) = policy
{
details.attempt_count < policy.max_attempts
&& elapsed.saturating_add(delay) < policy.max_elapsed
} else {
ordinary_retries += 1;
ordinary_retries <= 5
};
if !error.retryable() || !allowed {
return Err(error.with_retry_details(details.attempt_count, details.backoff));
}
if let Some(cancellation) = cancellation {
tokio::select! {
_ = cancellation.cancelled() => continue,
_ = clock.sleep(delay) => {}
}
} else {
clock.sleep(delay).await;
}
details.backoff += delay;
if error.throttled()
&& policy.is_some_and(|p| clock.elapsed().saturating_sub(started) >= p.max_elapsed)
{
return Err(error.with_retry_details(details.attempt_count, details.backoff));
}
}
}
const API_PATH_VERSION: &str = crate::constants::MODEL_API_VERSION;
pub const SIGNING_NAME: &str = "lambda";
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Call {
pub operation: &'static str,
pub method: Method,
pub path: String,
pub body: Option<Vec<u8>>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Method {
Get,
Post,
Patch,
Delete,
}
impl Method {
pub fn as_str(self) -> &'static str {
match self {
Method::Get => "GET",
Method::Post => "POST",
Method::Patch => "PATCH",
Method::Delete => "DELETE",
}
}
}
impl Call {
pub fn get(operation: &'static str, path: impl Into<String>) -> Self {
Self {
operation,
method: Method::Get,
path: path.into(),
body: None,
}
}
pub fn post_json<T: serde::Serialize>(
operation: &'static str,
path: impl Into<String>,
body: &T,
) -> Result<Self, Error> {
let body = serde_json::to_vec(body).map_err(|error| {
Error::new(
ErrorKind::Unexpected,
format!("could not serialize the {operation} request body: {error}"),
)
})?;
Ok(Self {
operation,
method: Method::Post,
path: path.into(),
body: Some(body),
})
}
pub fn patch_json<T: serde::Serialize>(
operation: &'static str,
path: impl Into<String>,
body: &T,
) -> Result<Self, Error> {
let body = serde_json::to_vec(body).map_err(|error| {
Error::new(
ErrorKind::Unexpected,
format!("could not serialize the {operation} request body: {error}"),
)
})?;
Ok(Self {
operation,
method: Method::Patch,
path: path.into(),
body: Some(body),
})
}
pub fn post_empty(operation: &'static str, path: impl Into<String>) -> Self {
Self {
operation,
method: Method::Post,
path: path.into(),
body: Some(b"{}".to_vec()),
}
}
pub fn delete(operation: &'static str, path: impl Into<String>) -> Self {
Self {
operation,
method: Method::Delete,
path: path.into(),
body: None,
}
}
pub fn body_bytes(&self) -> &[u8] {
self.body.as_deref().unwrap_or(&[])
}
}
#[derive(Clone, Debug)]
pub struct Reply {
pub status: u16,
pub body: Vec<u8>,
pub error_type: Option<String>,
}
impl Reply {
pub fn json<T: serde::de::DeserializeOwned>(&self, operation: &str) -> Result<T, Error> {
serde_json::from_slice(&self.body).map_err(|error| {
let text = String::from_utf8_lossy(&self.body);
let shown: String = text.chars().take(512).collect();
Error::new(
ErrorKind::Platform,
format!(
"could not read the {operation} response ({error}). The service model for \
{API_PATH_VERSION} is what this client's shapes are transcribed from, so a \
member that has moved is a drift-gate failure rather than a transport bug. \
Body: {shown}"
),
)
.with_source(error)
})
}
fn is_success(&self) -> bool {
(200..300).contains(&self.status)
}
}
pub trait Transport: Send + Sync {
fn region(&self) -> Option<&Region> {
None
}
fn send(
&self,
call: Call,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Reply, Error>> + Send + '_>>;
fn resolve_credentials(
&self,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), Error>> + Send + '_>> {
Box::pin(async { Ok(()) })
}
}
pub async fn send_with_retry(transport: &dyn Transport, call: Call) -> Result<Reply, Error> {
use backon::Retryable;
let operation = call.operation;
let attempt = || {
let call = call.clone();
async move {
let reply = transport.send(call).await?;
if reply.is_success() {
return Ok(reply);
}
Err(classify_failure_in_region(
operation,
&reply,
transport.region(),
))
}
};
let mut details = RetryDetails {
attempt_count: 1,
backoff: Duration::ZERO,
};
let result = attempt
.retry(retry_delays())
.when(Error::retryable)
.notify(|_, delay| {
details.attempt_count += 1;
details.backoff += delay;
})
.await;
result.map_err(|error| error.with_retry_details(details.attempt_count, details.backoff))
}
pub async fn send_accepting(
transport: &dyn Transport,
call: Call,
accept: &[u16],
) -> Result<Reply, Error> {
use backon::Retryable;
let operation = call.operation;
let attempt = || {
let call = call.clone();
async move {
let reply = transport.send(call).await?;
if reply.is_success() || accept.contains(&reply.status) {
return Ok(reply);
}
Err(classify_failure_in_region(
operation,
&reply,
transport.region(),
))
}
};
let mut details = RetryDetails {
attempt_count: 1,
backoff: Duration::ZERO,
};
let result = attempt
.retry(retry_delays())
.when(Error::retryable)
.notify(|_, delay| {
details.attempt_count += 1;
details.backoff += delay;
})
.await;
result.map_err(|error| error.with_retry_details(details.attempt_count, details.backoff))
}
pub fn classify_failure(operation: &str, reply: &Reply) -> Error {
classify_failure_in_region(operation, reply, None)
}
fn classify_failure_in_region(operation: &str, reply: &Reply, region: Option<&Region>) -> Error {
let parsed: crate::control::ops::ServiceErrorWire =
serde_json::from_slice(&reply.body).unwrap_or_default();
let detail = parsed.message.as_deref().unwrap_or("").trim().to_string();
let (kind, explanation) = match reply.status {
status if reply.error_type.as_deref() == Some("ThrottlingException") => (
ErrorKind::Retryable,
format!(
"ThrottlingException, named by the x-amzn-ErrorType header on an HTTP {status} \
— retry the identical request."
),
),
400 => (
ErrorKind::Platform,
"ValidationException — the service refused a value. Every constraint the pinned \
service model states on a member this client sends is checked locally before the \
call — min as well as max, pattern, and enum, because nothing else validates a \
request here: this client signs with aws-sigv4 and sends with reqwest, so botocore's \
VALIDATED_METADATA_ATTRS never applies to it. So reaching this status means a \
constraint this client does not yet check, and closing it is a guard in \
crates/microvms-app/src/control plus a line in tools/check-model-drift.py."
.to_string(),
),
402 => (
ErrorKind::Platform,
"ServiceQuotaExceededException — an account limit, not a request defect.".to_string(),
),
403 if detail.is_empty() => (
ErrorKind::Credentials,
null_message_denial(operation, region),
),
403 => (
ErrorKind::Credentials,
"AccessDeniedException, with a message — the service reported a genuine denial."
.to_string(),
),
404 => (
ErrorKind::Platform,
"ResourceNotFoundException — the image or MicroVM does not exist.".to_string(),
),
409 => (
ErrorKind::Platform,
"ConflictException — the resource is in a state that forbids this call. An image in \
CREATING refuses deletion, which is why teardown retries."
.to_string(),
),
429 => (
ErrorKind::Retryable,
"ThrottlingException — retry the identical request.".to_string(),
),
status if status >= 500 => (
ErrorKind::Retryable,
"InternalServerException — a service fault, retry the identical request.".to_string(),
),
_ => (
ErrorKind::Platform,
"an unmodeled status: the service model declares 400, 402, 403, 404, 409, 429, and \
500 for these operations."
.to_string(),
),
};
let said = if detail.is_empty() {
"the service sent no message".to_string()
} else {
format!("the service said {detail:?}")
};
Error::new(
kind,
format!(
"{operation} failed with HTTP {}: {said}. {explanation}",
reply.status
),
)
.with_throttled(
reply.status == 429 || reply.error_type.as_deref() == Some("ThrottlingException"),
)
}
fn null_message_denial(operation: &str, region: Option<&Region>) -> String {
let availability = match region {
Some(region) if region.is_measured() => format!(
"{} carries MicroVMs; an IAM policy is the likely cause.",
region.as_str()
),
Some(region) => format!(
"{} is not in the known MicroVMs region list; verify service availability as well as the IAM policy.",
region.as_str()
),
None => "The cause may be an unsupported region or an IAM denial.".to_string(),
};
let region_arg = region
.map(|region| format!(" --region {}", region.as_str()))
.unwrap_or_default();
format!(
"AccessDeniedException with an empty or null message field. {availability} \
A create call with tags can require the implied lambda:TagResource action. \
Check the full denial in CloudTrail: aws cloudtrail lookup-events{region_arg} \
--lookup-attributes AttributeKey=EventName,AttributeValue={operation} \
--query 'Events[].CloudTrailEvent' --output text. Inspect errorCode and errorMessage \
for the denied action (docs/PLATFORM.md)."
)
}
pub fn error_type_name(header: &str) -> &str {
let name = header.split(':').next().unwrap_or(header);
name.split_once('#').map_or(name, |(_, after)| after).trim()
}
pub fn endpoint_for(region: &Region) -> String {
format!("https://{SIGNING_NAME}.{}.amazonaws.com", region.as_str())
}
pub mod paths {
use super::API_PATH_VERSION;
pub fn encode_segment(segment: &str) -> String {
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
const ESCAPED: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
utf8_percent_encode(segment, ESCAPED).to_string()
}
pub fn microvm_images() -> String {
format!("/{API_PATH_VERSION}/microvm-images")
}
pub fn microvm_images_list(name_filter: Option<&str>, next_token: Option<&str>) -> String {
let mut query: Vec<String> = Vec::new();
if let Some(filter) = name_filter {
query.push(format!("nameFilter={}", encode_segment(filter)));
}
if let Some(token) = next_token {
query.push(format!("nextToken={}", encode_segment(token)));
}
if query.is_empty() {
return microvm_images();
}
format!("{}?{}", microvm_images(), query.join("&"))
}
pub fn microvm_image(image: &str) -> String {
format!(
"/{API_PATH_VERSION}/microvm-images/{}",
encode_segment(image)
)
}
fn with_next_token(path: String, next_token: Option<&str>) -> String {
match next_token {
Some(token) => format!("{path}?nextToken={}", encode_segment(token)),
None => path,
}
}
pub fn image_versions(image: &str, next_token: Option<&str>) -> String {
image_versions_paged(image, next_token, None)
}
pub fn image_versions_paged(
image: &str,
next_token: Option<&str>,
max_results: Option<u32>,
) -> String {
let mut query: Vec<String> = Vec::new();
if let Some(size) = max_results {
query.push(format!("maxResults={}", size.clamp(1, 50)));
}
if let Some(token) = next_token {
query.push(format!("nextToken={}", encode_segment(token)));
}
let path = format!("{}/versions", microvm_image(image));
if query.is_empty() {
return path;
}
format!("{path}?{}", query.join("&"))
}
pub fn image_version(image: &str, version: &str) -> String {
format!(
"{}/versions/{}",
microvm_image(image),
encode_segment(version)
)
}
pub fn image_build(image: &str, version: &str, build_id: &str) -> String {
format!(
"{}/builds/{}",
image_version(image, version),
encode_segment(build_id)
)
}
pub fn managed_microvm_images(next_token: Option<&str>) -> String {
with_next_token(
format!("/{API_PATH_VERSION}/managed-microvm-images"),
next_token,
)
}
pub fn managed_image_versions(image: &str, next_token: Option<&str>) -> String {
with_next_token(
format!(
"/{API_PATH_VERSION}/managed-microvm-images/{}/versions",
encode_segment(image)
),
next_token,
)
}
pub fn image_builds(image: &str, version: &str, next_token: Option<&str>) -> String {
with_next_token(
format!("{}/builds", image_version(image, version)),
next_token,
)
}
pub fn microvms() -> String {
format!("/{API_PATH_VERSION}/microvms")
}
pub fn microvms_list(next_token: Option<&str>) -> String {
with_next_token(microvms(), next_token)
}
pub fn microvms_list_matching(
image_identifier: Option<&str>,
image_version: Option<&str>,
next_token: Option<&str>,
) -> String {
let query: Vec<String> = [
("imageIdentifier", image_identifier),
("imageVersion", image_version),
("nextToken", next_token),
]
.into_iter()
.filter_map(|(key, value)| value.map(|value| format!("{key}={}", encode_segment(value))))
.collect();
if query.is_empty() {
return microvms();
}
format!("{}?{}", microvms(), query.join("&"))
}
pub fn microvm(id: &str) -> String {
format!("/{API_PATH_VERSION}/microvms/{}", encode_segment(id))
}
pub fn suspend(id: &str) -> String {
format!("{}/suspend", microvm(id))
}
pub fn resume(id: &str) -> String {
format!("{}/resume", microvm(id))
}
pub fn auth_token(id: &str) -> String {
format!("{}/auth-token", microvm(id))
}
pub fn shell_auth_token(id: &str) -> String {
format!("{}/shell-auth-token", microvm(id))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn throttle_budget_counts_attempts_bounds_elapsed_and_keeps_other_failures_default() {
use crate::clock::Clock;
use crate::control::fake::{Answer, FakeControlPlane, TestClock};
for (status, policy, expected) in [
(429, None, 6),
(
429,
Some(RetryPolicy::new(3, Duration::from_secs(3600)).unwrap()),
3,
),
(
429,
Some(RetryPolicy::new(9, Duration::from_millis(1)).unwrap()),
1,
),
(
503,
Some(RetryPolicy::new(9, Duration::from_secs(3600)).unwrap()),
6,
),
(
403,
Some(RetryPolicy::new(9, Duration::from_secs(3600)).unwrap()),
1,
),
] {
let transport = FakeControlPlane::new();
for _ in 0..expected {
transport.answer("RunMicrovm", Answer::failure(status, "failure"));
}
transport.answer("RunMicrovm", Answer::ok("{}"));
let clock = TestClock::new();
let error = send_configured(
&transport,
Call::get("RunMicrovm", "/run"),
&[],
policy,
&clock,
)
.await
.unwrap_err();
assert_eq!(transport.call_count("RunMicrovm"), expected);
assert_eq!(error.attempt_count(), expected as u32);
assert_eq!(error.throttled(), status == 429);
assert_eq!(error.backoff(), clock.elapsed());
if expected == 1 {
assert_eq!(clock.elapsed(), Duration::ZERO);
}
}
assert!(RetryPolicy::new(0, Duration::from_secs(1)).is_err());
assert!(RetryPolicy::new(1, Duration::ZERO).is_err());
}
#[tokio::test]
async fn a_retry_at_the_elapsed_budget_boundary_does_not_sleep() {
use crate::clock::Clock;
use crate::control::fake::{Answer, FakeControlPlane, TestClock};
let transport = FakeControlPlane::new();
transport.answer("RunMicrovm", Answer::throttled_as_400());
let gate = transport.hold_responses();
let clock = TestClock::new();
clock.set_unix_now(Duration::ZERO);
let pending = send_configured(
&transport,
Call::get("RunMicrovm", "/run"),
&[],
Some(RetryPolicy::new(2, Duration::MAX).unwrap()),
&clock,
);
tokio::pin!(pending);
tokio::select! {
biased;
_ = &mut pending => panic!("response must wait for the gate"),
_ = std::future::ready(()) => {}
}
assert_eq!(transport.call_count("RunMicrovm"), 1);
clock.advance(Duration::MAX);
gate.notify_one();
let error = tokio::select! {
biased;
result = &mut pending => result.unwrap_err(),
_ = std::future::ready(()) => panic!("elapsed budget must stop before another request"),
};
assert!(error.throttled());
assert_eq!(error.attempt_count(), 1);
assert_eq!(transport.call_count("RunMicrovm"), 1);
assert_eq!(error.backoff(), Duration::ZERO);
assert_eq!(clock.elapsed(), Duration::MAX);
}
#[tokio::test]
async fn configured_retries_recover_header_throttles_with_the_identical_call() {
use crate::clock::Clock;
use crate::control::fake::{Answer, FakeControlPlane, TestClock};
let transport = FakeControlPlane::new();
for _ in 0..7 {
transport.answer("RunMicrovm", Answer::throttled_as_400());
}
transport.answer("RunMicrovm", Answer::ok("{}"));
let clock = TestClock::new();
let call = Call::post_json(
"RunMicrovm",
"/run",
&serde_json::json!({"clientToken":"same-launch"}),
)
.unwrap();
let (_, details) = send_configured(
&transport,
call.clone(),
&[],
Some(RetryPolicy::new(8, Duration::from_secs(3600)).unwrap()),
&clock,
)
.await
.unwrap();
assert_eq!(details.attempt_count, 8);
assert_eq!(details.backoff, clock.elapsed());
assert!(details.backoff > Duration::ZERO);
assert!(transport.calls().iter().all(|sent| sent == &call));
transport.answer("GetMicrovm", Answer::failure(404, "missing"));
let (reply, details) = send_configured(
&transport,
Call::get("GetMicrovm", "/missing"),
&[404],
None,
&clock,
)
.await
.unwrap();
assert_eq!(reply.status, 404);
assert_eq!(details.attempt_count, 1);
assert_eq!(details.backoff, Duration::ZERO);
}
#[tokio::test(start_paused = true)]
async fn exhausted_public_retry_reports_throttle_attempts_and_backoff() {
use crate::control::fake::{Answer, FakeControlPlane};
for accepts_missing in [false, true] {
let transport = FakeControlPlane::new();
for _ in 0..6 {
transport.answer("ListMicrovms", Answer::throttled_as_400());
}
transport.answer("ListMicrovms", Answer::ok("{}"));
let call = Call::get("ListMicrovms", "/list");
let error = if accepts_missing {
send_accepting(&transport, call, &[404]).await
} else {
send_with_retry(&transport, call).await
}
.unwrap_err();
assert_eq!(transport.call_count("ListMicrovms"), 6);
assert!(error.throttled());
assert_eq!(error.attempt_count(), 6);
assert!(error.backoff() >= Duration::from_millis(6200));
assert!(error.backoff() < Duration::from_millis(12400));
}
for status in [200, 404] {
let transport = FakeControlPlane::new();
transport.answer("GetMicrovm", Answer::failure(status, "accepted"));
let reply = send_accepting(&transport, Call::get("GetMicrovm", "/missing"), &[404])
.await
.unwrap();
assert_eq!(reply.status, status);
assert_eq!(transport.call_count("GetMicrovm"), 1);
}
}
#[tokio::test]
async fn a_cancel_signal_wakes_backoff_before_another_request() {
use crate::clock::testing::YieldingClock;
use crate::control::fake::{Answer, FakeControlPlane};
let transport = FakeControlPlane::new();
transport.answer("RunMicrovm", Answer::throttled_as_400());
let clock = YieldingClock::new();
let cancellation = RetryCancellation::default();
let call = Call::get("RunMicrovm", "/run");
let pending = send_cancellable(
&transport,
call,
&[],
Some(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap()),
&clock,
Some(&cancellation),
);
tokio::pin!(pending);
tokio::select! { biased; _ = &mut pending => panic!("backoff should yield"), _ = std::future::ready(()) => {} }
cancellation.cancel();
let error = pending.await.unwrap_err();
assert_eq!(error.kind(), ErrorKind::Interrupted);
assert_eq!(transport.call_count("RunMicrovm"), 1);
}
#[tokio::test]
async fn dropping_a_retry_future_prevents_another_request() {
use crate::clock::testing::YieldingClock;
use crate::control::fake::{Answer, FakeControlPlane};
let transport = FakeControlPlane::new();
transport.answer("RunMicrovm", Answer::throttled_as_400());
let clock = YieldingClock::new();
let mut future = Box::pin(send_configured(
&transport,
Call::get("RunMicrovm", "/run"),
&[],
Some(RetryPolicy::new(100, Duration::from_secs(3600)).unwrap()),
&clock,
));
tokio::select! { biased; _ = &mut future => panic!("retry sleep should yield"), _ = std::future::ready(()) => {} }
drop(future);
clock.advance(Duration::from_secs(3600));
tokio::task::yield_now().await;
tokio::task::yield_now().await;
assert_eq!(transport.call_count("RunMicrovm"), 1);
}
#[test]
fn the_endpoint_uses_the_models_lambda_prefix_for_every_region() {
assert_eq!(
endpoint_for(&Region::US_EAST_1),
"https://lambda.us-east-1.amazonaws.com"
);
assert_eq!(
endpoint_for(&Region::AP_NORTHEAST_1),
"https://lambda.ap-northeast-1.amazonaws.com"
);
assert_eq!(
endpoint_for(&"us-west-1".parse().expect("a region name")),
"https://lambda.us-west-1.amazonaws.com"
);
}
#[test]
fn every_path_matches_the_models_request_uri() {
assert_eq!(paths::microvm_images(), "/2025-09-09/microvm-images");
assert_eq!(
paths::microvm_image("img"),
"/2025-09-09/microvm-images/img"
);
assert_eq!(
paths::image_versions("img", None),
"/2025-09-09/microvm-images/img/versions"
);
assert_eq!(
paths::image_version("img", "1"),
"/2025-09-09/microvm-images/img/versions/1"
);
assert_eq!(
paths::image_builds("img", "1", None),
"/2025-09-09/microvm-images/img/versions/1/builds"
);
assert_eq!(
paths::image_build("img", "1", "build-abc"),
"/2025-09-09/microvm-images/img/versions/1/builds/build-abc"
);
assert_eq!(
paths::managed_microvm_images(None),
"/2025-09-09/managed-microvm-images"
);
assert_eq!(
paths::managed_image_versions(
"arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1",
None
),
"/2025-09-09/managed-microvm-images/\
arn%3Aaws%3Alambda%3Aus-east-1%3Aaws%3Amicrovm-image%3Aal2023-1/versions"
);
assert_eq!(paths::microvms(), "/2025-09-09/microvms");
assert_eq!(paths::microvms_list(None), "/2025-09-09/microvms");
assert_eq!(paths::microvm("mvm-1"), "/2025-09-09/microvms/mvm-1");
assert_eq!(
paths::suspend("mvm-1"),
"/2025-09-09/microvms/mvm-1/suspend"
);
assert_eq!(paths::resume("mvm-1"), "/2025-09-09/microvms/mvm-1/resume");
assert_eq!(
paths::auth_token("mvm-1"),
"/2025-09-09/microvms/mvm-1/auth-token"
);
assert_eq!(
paths::shell_auth_token("mvm-1"),
"/2025-09-09/microvms/mvm-1/shell-auth-token"
);
}
#[test]
fn the_image_listing_path_encodes_its_query_members() {
assert_eq!(
paths::microvm_images_list(None, None),
"/2025-09-09/microvm-images"
);
assert_eq!(
paths::microvm_images_list(Some("my-image"), None),
"/2025-09-09/microvm-images?nameFilter=my-image"
);
assert_eq!(
paths::microvm_images_list(Some("my-image"), Some("a+b/c=")),
"/2025-09-09/microvm-images?nameFilter=my-image&nextToken=a%2Bb%2Fc%3D"
);
assert_eq!(
paths::microvm_images_list(None, Some("token")),
"/2025-09-09/microvm-images?nextToken=token"
);
}
#[test]
fn the_other_paginated_listings_encode_their_cursor_too() {
assert_eq!(
paths::image_versions("img", Some("a+b/c=")),
"/2025-09-09/microvm-images/img/versions?nextToken=a%2Bb%2Fc%3D"
);
assert_eq!(
paths::image_builds("img", "1", Some("a+b/c=")),
"/2025-09-09/microvm-images/img/versions/1/builds?nextToken=a%2Bb%2Fc%3D"
);
assert_eq!(
paths::microvms_list(Some("a+b/c=")),
"/2025-09-09/microvms?nextToken=a%2Bb%2Fc%3D"
);
for path in [
paths::image_versions("img", None),
paths::image_builds("img", "1", None),
paths::microvms_list(None),
] {
assert!(
!path.contains('?'),
"an absent cursor emits no query: {path}"
);
}
}
#[test]
fn the_version_listings_explicit_page_size_is_ordered_and_clamped() {
assert_eq!(
paths::image_versions_paged("img", None, Some(1)),
"/2025-09-09/microvm-images/img/versions?maxResults=1"
);
assert_eq!(
paths::image_versions_paged("img", Some("a+b/c="), Some(2)),
"/2025-09-09/microvm-images/img/versions?maxResults=2&nextToken=a%2Bb%2Fc%3D"
);
assert_eq!(
paths::image_versions_paged("img", None, Some(0)),
"/2025-09-09/microvm-images/img/versions?maxResults=1",
"the model's min is 1; a 0 would be refused"
);
assert_eq!(
paths::image_versions_paged("img", None, Some(999)),
"/2025-09-09/microvm-images/img/versions?maxResults=50",
"the model's max is 50; a larger value would be refused"
);
assert_eq!(
paths::image_versions_paged("img", None, None),
paths::image_versions("img", None)
);
assert!(
!paths::image_versions("img", Some("t")).contains("maxResults"),
"no production path names a page size: {}",
paths::image_versions("img", Some("t"))
);
}
#[test]
fn the_managed_routes_are_their_own_collection_and_encode_their_cursor() {
assert!(
!paths::managed_microvm_images(None).contains('?'),
"an absent cursor emits no query member"
);
assert_eq!(
paths::managed_microvm_images(Some("a+b/c=")),
"/2025-09-09/managed-microvm-images?nextToken=a%2Bb%2Fc%3D"
);
assert_eq!(
paths::managed_image_versions("al2023-1", Some("a+b/c=")),
"/2025-09-09/managed-microvm-images/al2023-1/versions?nextToken=a%2Bb%2Fc%3D"
);
assert_ne!(
paths::managed_microvm_images(None),
paths::microvm_images_list(None, None)
);
assert!(
!paths::microvm_images().contains("managed"),
"the account's own collection must not acquire the managed prefix"
);
}
#[test]
fn one_builds_path_is_the_listing_plus_an_encoded_segment() {
let path = paths::image_build("img", "1", "4a4c5e30-811f-47fa-9893-260ea6a37a8f");
assert_eq!(
path,
"/2025-09-09/microvm-images/img/versions/1/builds/4a4c5e30-811f-47fa-9893-260ea6a37a8f"
);
assert!(!path.contains('?'), "{path}");
assert!(
path.starts_with(&paths::image_builds("img", "1", None)),
"the get path must extend the list path rather than be a second route: {path}"
);
assert_eq!(
paths::image_build("img", "1", "a/b"),
"/2025-09-09/microvm-images/img/versions/1/builds/a%2Fb"
);
}
#[test]
fn deleting_one_version_carries_no_pagination_cursor() {
let path = paths::image_version("img", "1");
assert_eq!(path, "/2025-09-09/microvm-images/img/versions/1");
assert!(!path.contains('?'), "{path}");
assert!(!path.contains("nextToken"), "{path}");
}
#[test]
fn a_real_arn_identifier_is_percent_encoded_colons_and_all() {
let arn = "arn:aws:lambda:us-east-1:123456789012:microvm-image:img";
let path = paths::microvm_image(arn);
assert_eq!(
path,
"/2025-09-09/microvm-images/arn%3Aaws%3Alambda%3Aus-east-1%3A123456789012%3Amicrovm-image%3Aimg"
);
assert!(
!path["/2025-09-09/microvm-images/".len()..].contains(':'),
"an unencoded colon in a path segment is a signature mismatch: {path}"
);
}
#[test]
fn the_segment_encoder_encodes_slashes_wherever_they_come_from() {
assert_eq!(paths::encode_segment("a/b"), "a%2Fb");
let path = paths::microvm_image("has/a/slash");
assert_eq!(path, "/2025-09-09/microvm-images/has%2Fa%2Fslash");
assert!(
!path["/2025-09-09/microvm-images/".len()..].contains('/'),
"an encoded identifier contributes no path separators: {path}"
);
}
#[test]
fn the_path_version_is_the_pinned_model_version() {
assert_eq!(API_PATH_VERSION, crate::constants::MODEL_API_VERSION);
assert!(paths::microvms().starts_with("/2025-09-09/"));
}
#[test]
fn the_signing_name_is_lambda_not_lambda_microvms() {
assert_eq!(SIGNING_NAME, "lambda");
}
#[test]
fn a_null_message_access_denied_names_both_possible_causes() {
let reply = Reply {
status: 403,
body: br#"{"message": null}"#.to_vec(),
error_type: None,
};
let error = classify_failure("ListMicrovms", &reply);
assert_eq!(error.kind(), ErrorKind::Credentials);
let message = error.to_string();
assert!(message.contains("null"), "{message}");
assert!(message.contains("region"), "{message}");
assert!(
message.contains("unsupported region or an IAM denial"),
"{message}"
);
assert!(message.contains("lambda:TagResource"), "{message}");
assert!(message.contains("cloudtrail lookup-events"), "{message}");
}
#[tokio::test]
async fn supported_region_null_denials_point_to_iam_in_both_send_paths() {
use crate::control::fake::{Answer, FakeControlPlane};
let transport = FakeControlPlane::in_region(Region::US_EAST_1);
transport.answer(
"CreateMicrovmImage",
Answer {
status: 403,
body: r#"{"message":null}"#.to_string(),
error_type: Some("AccessDeniedException".to_string()),
},
);
for accept in [false, true] {
let call = Call::get("CreateMicrovmImage", "/images");
let error = if accept {
send_accepting(&transport, call, &[404]).await.unwrap_err()
} else {
send_with_retry(&transport, call).await.unwrap_err()
};
assert_eq!(error.kind(), ErrorKind::Credentials);
assert!(!error.retryable());
let message = error.to_string();
for detail in [
"us-east-1 carries MicroVMs",
"policy is the likely cause",
"lambda:TagResource",
"cloudtrail lookup-events --region us-east-1",
"AttributeValue=CreateMicrovmImage",
"errorMessage",
] {
assert!(message.contains(detail), "{message}");
}
assert!(!message.contains("Check the region before the policy"));
}
assert_eq!(transport.call_count("CreateMicrovmImage"), 2);
}
#[test]
fn unlisted_region_null_denials_keep_both_causes_visible() {
let reply = Reply {
status: 403,
body: b"{}".to_vec(),
error_type: None,
};
let error = classify_failure_in_region(
"ListMicrovms",
&reply,
Some(&"us-west-1".parse().expect("a region name")),
);
let message = error.to_string();
assert!(message.contains("verify service availability as well as the IAM policy"));
assert!(message.contains("--region us-west-1"));
assert!(!message.contains("policy is the likely cause"));
}
#[test]
fn a_403_with_a_message_is_reported_as_a_genuine_denial() {
let reply = Reply {
status: 403,
body: br#"{"message": "User: arn:aws:iam::1:user/u is not authorized"}"#.to_vec(),
error_type: None,
};
let error = classify_failure("RunMicrovm", &reply);
assert_eq!(error.kind(), ErrorKind::Credentials);
let message = error.to_string();
assert!(message.contains("not authorized"), "{message}");
assert!(
message.contains("genuine denial"),
"must distinguish itself from the null-message case: {message}"
);
assert!(
!message.contains("does not carry MicroVMs"),
"must not blame the region when the service named a principal: {message}"
);
}
#[test]
fn a_400_names_the_validation_refusal_and_the_local_checks_it_passed() {
let reply = Reply {
status: 400,
body: br#"{"message": "1 validation error detected"}"#.to_vec(),
error_type: None,
};
let error = classify_failure("RunMicrovm", &reply);
assert_eq!(error.kind(), ErrorKind::Platform);
let message = error.to_string();
assert!(message.contains("ValidationException"), "{message}");
assert!(message.contains("checked locally"), "{message}");
assert!(!message.contains("unmodeled status"), "{message}");
}
#[test]
fn a_throttle_named_by_the_error_type_header_is_retryable_whatever_the_status() {
let reply = Reply {
status: 400,
body: br#"{"message":null,"throttlingReasons":null}"#.to_vec(),
error_type: Some("ThrottlingException".to_string()),
};
let error = classify_failure("ListMicrovms", &reply);
assert!(error.retryable(), "{error}");
assert!(error.throttled(), "the header identifies the throttle");
assert_eq!(error.kind(), ErrorKind::Retryable);
let message = error.to_string();
assert!(message.contains("ThrottlingException"), "{message}");
assert!(message.contains("HTTP 400"), "{message}");
assert!(!message.contains("ValidationException"), "{message}");
}
#[test]
fn a_400_naming_another_error_type_is_still_a_validation_refusal() {
let reply = Reply {
status: 400,
body: br#"{"message": "1 validation error detected"}"#.to_vec(),
error_type: Some("ValidationException".to_string()),
};
let error = classify_failure("RunMicrovm", &reply);
assert!(!error.retryable(), "{error}");
assert!(error.to_string().contains("checked locally"), "{error}");
}
#[test]
fn the_error_type_name_drops_the_uri_suffix_and_the_namespace() {
let cases = [
(
"ThrottlingException:http://internal.amazon.com/coral/com.amazon.coral.availability/",
"ThrottlingException",
),
(
"aws.protocoltests.restjson#FooError:http://internal.amazon.com/coral/com.amazon.coral.validate/",
"FooError",
),
(
"com.amazonaws.lambda#ResourceNotFoundException",
"ResourceNotFoundException",
),
("ConflictException", "ConflictException"),
];
for (header, name) in cases {
assert_eq!(error_type_name(header), name, "{header}");
}
}
#[test]
fn only_throttles_and_server_faults_are_retryable() {
let cases = [
(400, false),
(402, false),
(403, false),
(404, false),
(409, false),
(429, true),
(500, true),
(502, true),
(503, true),
];
for (status, retryable) in cases {
let reply = Reply {
status,
body: br#"{"message": "detail"}"#.to_vec(),
error_type: None,
};
let error = classify_failure("GetMicrovm", &reply);
assert_eq!(error.retryable(), retryable, "HTTP {status}");
assert_eq!(error.throttled(), status == 429, "HTTP {status}");
}
}
#[test]
fn a_control_plane_failure_carries_no_daemon_wire_kind() {
for status in [400, 403, 404, 409, 429, 500] {
let reply = Reply {
status,
body: Vec::new(),
error_type: None,
};
assert_eq!(
classify_failure("GetMicrovm", &reply).wire_kind(),
None,
"HTTP {status} never reached the daemon"
);
}
}
#[test]
fn an_unparseable_error_body_still_classifies_by_status() {
let reply = Reply {
status: 429,
body: b"<html>Too Many Requests</html>".to_vec(),
error_type: None,
};
let error = classify_failure("RunMicrovm", &reply);
assert!(error.retryable(), "a throttle is a throttle");
assert!(error.to_string().contains("no message"), "{error}");
}
#[test]
fn a_response_that_does_not_parse_reports_the_body() {
let reply = Reply {
status: 200,
body: br#"{"microvmId": 42}"#.to_vec(),
error_type: None,
};
let error = reply
.json::<crate::control::ops::MicrovmResponseWire>("GetMicrovm")
.expect_err("microvmId is a string shape");
let message = error.to_string();
assert!(message.contains("GetMicrovm"), "{message}");
assert!(message.contains("microvmId"), "{message}");
assert_eq!(error.kind(), ErrorKind::Platform);
}
#[test]
fn a_bodyless_post_sends_an_empty_json_object() {
let call = Call::post_empty("SuspendMicrovm", paths::suspend("mvm-1"));
assert_eq!(call.body.as_deref(), Some(&b"{}"[..]));
assert_eq!(call.method, Method::Post);
}
#[test]
fn a_get_and_a_delete_carry_no_body() {
let get = Call::get("GetMicrovm", paths::microvm("mvm-1"));
assert_eq!(get.body, None);
assert_eq!(get.body_bytes(), b"");
let delete = Call::delete("TerminateMicrovm", paths::microvm("mvm-1"));
assert_eq!(delete.body, None);
assert_eq!(delete.method, Method::Delete);
}
#[test]
fn the_methods_are_the_ones_the_model_uses_spelled_uppercase() {
assert_eq!(Method::Get.as_str(), "GET");
assert_eq!(Method::Post.as_str(), "POST");
assert_eq!(Method::Patch.as_str(), "PATCH");
assert_eq!(Method::Delete.as_str(), "DELETE");
}
#[test]
fn a_patch_carries_its_body_and_is_not_a_post() {
let call = Call::patch_json(
"UpdateMicrovmImageVersion",
paths::image_version("img", "2.0"),
&crate::control::ops::UpdateImageVersionWire {
status: crate::control::ops::VersionStatus::Inactive,
},
)
.expect("serialises");
assert_eq!(call.method, Method::Patch);
assert_ne!(
call.method,
Method::Post,
"a POST to the version path is a route the model does not declare"
);
assert_eq!(call.path, "/2025-09-09/microvm-images/img/versions/2.0");
assert_eq!(
call.body.as_deref(),
Some(&br#"{"status":"INACTIVE"}"#[..]),
"the one member the request has must reach the body"
);
assert_eq!(call.operation, "UpdateMicrovmImageVersion");
}
}