use std::pin::Pin;
use std::sync::Arc;
#[cfg(test)]
use std::sync::Mutex;
use aws_sdk_lambda::types::{Operation, OperationType, OperationUpdate};
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum ExecutionClientErrorScope {
Invocation,
Execution,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct ExecutionClientError {
scope: ExecutionClientErrorScope,
message: String,
source: Option<Arc<dyn std::error::Error + Send + Sync>>,
}
impl std::fmt::Display for ExecutionClientError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "durable: client: {}", self.message)
}
}
impl std::error::Error for ExecutionClientError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source
.as_deref()
.map(|s| s as &(dyn std::error::Error + 'static))
}
}
impl ExecutionClientError {
#[must_use]
pub fn new(scope: ExecutionClientErrorScope, message: impl Into<String>) -> Self {
Self {
scope,
message: message.into(),
source: None,
}
}
#[must_use]
pub fn with_source(
scope: ExecutionClientErrorScope,
message: impl Into<String>,
source: impl Into<crate::BoxError>,
) -> Self {
Self {
scope,
message: message.into(),
source: Some(Arc::from(source.into())),
}
}
pub fn scope(&self) -> ExecutionClientErrorScope {
self.scope
}
pub(crate) fn is_retryable(&self) -> bool {
self.scope == ExecutionClientErrorScope::Invocation
}
pub(crate) fn message(&self) -> &str {
&self.message
}
#[cfg(test)]
pub(crate) fn non_retryable(message: String) -> Self {
Self::new(ExecutionClientErrorScope::Execution, message)
}
pub(crate) fn new_non_retryable(message: &str) -> Self {
Self::new(ExecutionClientErrorScope::Execution, message)
}
#[cfg(any(test, feature = "test-util"))]
pub(crate) fn from_retryable(message: String) -> Self {
Self::new(ExecutionClientErrorScope::Invocation, message)
}
}
pub(crate) fn client_error_record_message(err: &ExecutionClientError) -> String {
use aws_sdk_lambda::error::ProvideErrorMetadata;
use std::error::Error as _;
if let Some(source) = err.source() {
if let Some(sdk) = source.downcast_ref::<CheckpointSdkError>()
&& let Some(m) = sdk.message()
{
return m.to_owned();
}
if let Some(sdk) = source.downcast_ref::<GetStateSdkError>()
&& let Some(m) = sdk.message()
{
return m.to_owned();
}
}
err.message().to_owned()
}
pub(crate) fn bootstrap_error_type(err: &ExecutionClientError) -> String {
use aws_sdk_lambda::error::ProvideErrorMetadata;
use std::error::Error as _;
err.source()
.and_then(|s| s.downcast_ref::<GetStateSdkError>())
.and_then(ProvideErrorMetadata::code)
.unwrap_or(crate::error::EXECUTION_CLIENT_ERROR_TYPE)
.to_owned()
}
type CheckpointSdkError = aws_sdk_lambda::error::SdkError<
aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError,
aws_sdk_lambda::config::http::HttpResponse,
>;
type GetStateSdkError = aws_sdk_lambda::error::SdkError<
aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError,
aws_sdk_lambda::config::http::HttpResponse,
>;
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CheckpointOutput {
pub(crate) checkpoint_token: Option<String>,
pub(crate) updated_operations: Vec<Operation>,
pub(crate) next_marker: Option<String>,
}
impl CheckpointOutput {
#[must_use]
pub fn new(
checkpoint_token: Option<String>,
updated_operations: Vec<Operation>,
next_marker: Option<String>,
) -> Self {
Self {
checkpoint_token,
updated_operations,
next_marker,
}
}
pub fn checkpoint_token(&self) -> Option<&str> {
self.checkpoint_token.as_deref().filter(|t| !t.is_empty())
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct GetStateOutput {
pub(crate) operations: Vec<Operation>,
pub(crate) next_marker: Option<String>,
}
impl GetStateOutput {
#[must_use]
pub fn new(operations: Vec<Operation>, next_marker: Option<String>) -> Self {
Self {
operations,
next_marker,
}
}
pub fn next_marker(&self) -> Option<&str> {
self.next_marker.as_deref().filter(|m| !m.is_empty())
}
}
pub trait ExecutionClient: Send + Sync {
fn checkpoint(
&self,
execution_arn: &str,
checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>;
fn get_state(
&self,
execution_arn: &str,
checkpoint_token: &str,
marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>;
}
const GET_STATE_MAX_ITEMS: i32 = 1000;
const USER_AGENT_TOKEN: &str =
concat!("aws-durable-execution-sdk-rust/", env!("CARGO_PKG_VERSION"));
#[derive(Debug)]
struct UserAgentTokenInterceptor;
impl aws_sdk_lambda::config::Intercept for UserAgentTokenInterceptor {
fn name(&self) -> &'static str {
"UserAgentTokenInterceptor"
}
fn modify_before_transmit(
&self,
context: &mut aws_sdk_lambda::config::interceptors::BeforeTransmitInterceptorContextMut<'_>,
_runtime_components: &aws_sdk_lambda::config::RuntimeComponents,
_cfg: &mut aws_sdk_lambda::config::ConfigBag,
) -> Result<(), aws_sdk_lambda::error::BoxError> {
let headers = context.request_mut().headers_mut();
let value = user_agent_with_token(headers.get("user-agent"));
headers.insert("user-agent", value);
Ok(())
}
}
fn user_agent_with_token(current: Option<&str>) -> String {
match current {
Some(current) => format!("{current} {USER_AGENT_TOKEN}"),
None => USER_AGENT_TOKEN.to_owned(),
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct LambdaExecutionClient {
client: aws_sdk_lambda::Client,
}
impl std::fmt::Debug for LambdaExecutionClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("LambdaExecutionClient")
}
}
impl LambdaExecutionClient {
#[must_use]
pub fn new(client: crate::LambdaClient) -> Self {
Self { client }
}
}
impl ExecutionClient for LambdaExecutionClient {
fn checkpoint(
&self,
execution_arn: &str,
checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let arn = execution_arn.to_owned();
let token = checkpoint_token.to_owned();
Box::pin(async move {
let result = self
.client
.checkpoint_durable_execution()
.durable_execution_arn(&arn)
.checkpoint_token(&token)
.set_updates(Some(updates))
.customize()
.interceptor(UserAgentTokenInterceptor)
.send()
.await;
match result {
Ok(output) => {
let new_token = output.checkpoint_token.filter(|t| !t.is_empty());
let (updated_ops, next_marker) = match output.new_execution_state {
Some(state) => (
state.operations.unwrap_or_default(),
state.next_marker.filter(|m| !m.is_empty()),
),
None => (Vec::new(), None),
};
Ok(CheckpointOutput::new(new_token, updated_ops, next_marker))
}
Err(err) => Err(classify_checkpoint_error(err)),
}
})
}
fn get_state(
&self,
execution_arn: &str,
checkpoint_token: &str,
marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
let arn = execution_arn.to_owned();
let token = checkpoint_token.to_owned();
let marker = marker.map(str::to_owned);
Box::pin(async move {
let output = self
.client
.get_durable_execution_state()
.durable_execution_arn(&arn)
.checkpoint_token(&token)
.set_marker(marker)
.max_items(GET_STATE_MAX_ITEMS)
.customize()
.interceptor(UserAgentTokenInterceptor)
.send()
.await
.map_err(classify_get_state_error)?;
Ok(GetStateOutput::new(
output.operations,
output.next_marker.filter(|m| !m.is_empty()),
))
})
}
}
pub(crate) async fn get_full_state(
client: &dyn ExecutionClient,
execution_arn: &str,
checkpoint_token: &str,
) -> Result<Vec<Operation>, ExecutionClientError> {
let mut operations = Vec::new();
let mut marker: Option<String> = None;
loop {
let page = client
.get_state(execution_arn, checkpoint_token, marker.as_deref())
.await?;
let next = page.next_marker().map(str::to_owned);
operations.extend(page.operations);
match next {
Some(m) => marker = Some(m),
None => return Ok(operations),
}
}
}
const STALE_TOKEN_MESSAGE_PREFIX: &str = "Invalid checkpoint token";
fn classify_checkpoint_error(err: CheckpointSdkError) -> ExecutionClientError {
let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
let scope = checkpoint_error_scope(&err);
ExecutionClientError::with_source(scope, message, err)
}
fn checkpoint_error_scope(err: &CheckpointSdkError) -> ExecutionClientErrorScope {
use ExecutionClientErrorScope::{Execution, Invocation};
use aws_sdk_lambda::error::SdkError;
use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as E;
if let SdkError::ServiceError(service_err) = err {
match service_err.err() {
E::TooManyRequestsException(_) | E::ServiceException(_) => return Invocation,
E::InvalidParameterValueException(inner) => {
return if inner
.message()
.is_some_and(|m| m.starts_with(STALE_TOKEN_MESSAGE_PREFIX))
{
Invocation
} else {
Execution
};
}
E::KmsAccessDeniedException(_)
| E::KmsDisabledException(_)
| E::KmsInvalidStateException(_)
| E::KmsNotFoundException(_) => return Execution,
_ => {}
}
}
match err.raw_response().map(|r| r.status().as_u16()) {
Some(status) if (400..500).contains(&status) && status != 429 => Execution,
_ => Invocation,
}
}
fn classify_get_state_error(err: GetStateSdkError) -> ExecutionClientError {
use aws_sdk_lambda::operation::get_durable_execution_state::GetDurableExecutionStateError as E;
let message = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(&err));
let scope = match err.as_service_error() {
Some(
E::KmsAccessDeniedException(_)
| E::KmsDisabledException(_)
| E::KmsInvalidStateException(_)
| E::KmsNotFoundException(_),
) => ExecutionClientErrorScope::Execution,
_ => ExecutionClientErrorScope::Invocation,
};
ExecutionClientError::with_source(scope, message, err)
}
#[cfg(test)]
pub(crate) fn mock_aws_error_response(
status: u16,
code: &str,
message: &str,
) -> aws_sdk_lambda::config::http::HttpResponse {
let body = serde_json::json!({ "Type": "User", "message": message }).to_string();
let mut response = aws_sdk_lambda::config::http::HttpResponse::new(
aws_smithy_runtime_api::http::StatusCode::try_from(status).expect("valid HTTP status"),
aws_smithy_types::body::SdkBody::from(body),
);
response
.headers_mut()
.insert("x-amzn-errortype", code.to_owned());
response
.headers_mut()
.insert("content-type", "application/json");
response
}
#[cfg(test)]
#[derive(Debug, Clone)]
pub(crate) enum TestResponse {
Success(Vec<Operation>),
SuccessPaginated(Vec<Operation>, String),
RetryableError(String),
NonRetryableError(String),
AcceptThenRetryableError(String),
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct InMemoryExecutionClient {
state_operations: Mutex<Vec<Operation>>,
checkpoint_responses: Mutex<Vec<TestResponse>>,
pub(crate) checkpoint_call_count: Mutex<u32>,
pub(crate) get_state_call_count: Mutex<u32>,
get_state_failure: Mutex<Option<String>>,
token_counter: Mutex<u32>,
callback_counter: Mutex<u32>,
recorded_updates: Mutex<Vec<OperationUpdate>>,
}
#[cfg(test)]
impl InMemoryExecutionClient {
pub(crate) fn new(state_operations: Vec<Operation>) -> Self {
Self {
state_operations: Mutex::new(state_operations),
checkpoint_responses: Mutex::new(Vec::new()),
checkpoint_call_count: Mutex::new(0),
get_state_call_count: Mutex::new(0),
get_state_failure: Mutex::new(None),
token_counter: Mutex::new(0),
callback_counter: Mutex::new(0),
recorded_updates: Mutex::new(Vec::new()),
}
}
pub(crate) fn enqueue_checkpoint_response(&self, response: TestResponse) {
let mut responses = self
.checkpoint_responses
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
responses.push(response);
}
pub(crate) fn fail_get_state(&self, message: &str) {
let mut failure = self
.get_state_failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*failure = Some(message.to_owned());
}
pub(crate) fn recorded_updates(&self) -> Vec<OperationUpdate> {
self.recorded_updates
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
}
#[cfg(test)]
impl ExecutionClient for InMemoryExecutionClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
{
let mut count = self
.checkpoint_call_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*count += 1;
}
let auto_ops: Vec<Operation> = updates
.iter()
.filter(|u| {
u.r#type() == &OperationType::Callback
&& u.action() == &aws_sdk_lambda::types::OperationAction::Start
})
.map(|u| {
let mut counter = self
.callback_counter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*counter += 1;
Operation::builder()
.id(u.id())
.r#type(OperationType::Callback)
.status(aws_sdk_lambda::types::OperationStatus::Pending)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.callback_details(
aws_sdk_lambda::types::CallbackDetails::builder()
.callback_id(format!("in-mem-cb-{counter}"))
.build(),
)
.build()
.expect("all required Operation fields set")
})
.collect();
{
let mut recorded = self
.recorded_updates
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
recorded.extend(updates);
}
let response = {
let mut responses = self
.checkpoint_responses
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if responses.is_empty() {
None
} else {
Some(responses.remove(0))
}
};
Box::pin(async move {
match response {
Some(
TestResponse::RetryableError(msg) | TestResponse::AcceptThenRetryableError(msg),
) => Err(ExecutionClientError::from_retryable(msg)),
Some(TestResponse::NonRetryableError(msg)) => {
Err(ExecutionClientError::non_retryable(msg))
}
Some(TestResponse::Success(ops)) => {
let mut counter = self
.token_counter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*counter += 1;
Ok(CheckpointOutput {
checkpoint_token: Some(format!("token-{counter}")),
updated_operations: ops,
next_marker: None,
})
}
Some(TestResponse::SuccessPaginated(ops, marker)) => {
let mut counter = self
.token_counter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*counter += 1;
Ok(CheckpointOutput {
checkpoint_token: Some(format!("token-{counter}")),
updated_operations: ops,
next_marker: Some(marker),
})
}
None => {
let mut counter = self
.token_counter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*counter += 1;
Ok(CheckpointOutput {
checkpoint_token: Some(format!("token-{counter}")),
updated_operations: auto_ops,
next_marker: None,
})
}
}
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
{
let mut count = self
.get_state_call_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*count += 1;
}
let failure = self
.get_state_failure
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
if let Some(message) = failure {
return Box::pin(async move { Err(ExecutionClientError::non_retryable(message)) });
}
let ops = self
.state_operations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
Box::pin(async move { Ok(GetStateOutput::new(ops, None)) })
}
}
fn operation_to_record(op: &Operation) -> (String, CheckpointRecord) {
let attempt = op.step_details.as_ref().map_or(0, |s| {
#[expect(clippy::cast_sign_loss)] {
s.attempt.max(0) as u32
}
});
let record = CheckpointRecord {
id: op.id().to_owned(),
status: sdk_status_to_checkpoint_status(op.status()),
result: extract_result(op),
error_type: extract_error_type(op),
error_message: extract_error_message(op),
error_data: extract_wire_error_field(op, |e| e.error_data.clone()),
stack_trace: extract_wire_error_field(op, |e| {
(!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())
}),
attempt,
invoke_result: op
.chained_invoke_details
.as_ref()
.and_then(|i| i.result.clone()),
invoke_error_type: op
.chained_invoke_details
.as_ref()
.and_then(|i| i.error.as_ref())
.and_then(|e| e.error_type.clone()),
invoke_error_message: op
.chained_invoke_details
.as_ref()
.and_then(|i| i.error.as_ref())
.and_then(|e| e.error_message.clone()),
invoke_error_data: op
.chained_invoke_details
.as_ref()
.and_then(|i| i.error.as_ref())
.and_then(|e| e.error_data.clone()),
invoke_stack_trace: op
.chained_invoke_details
.as_ref()
.and_then(|i| i.error.as_ref())
.and_then(|e| (!e.stack_trace().is_empty()).then(|| e.stack_trace().to_vec())),
replay_children: op
.context_details
.as_ref()
.and_then(|c| c.replay_children)
.unwrap_or(false),
callback_id: op
.callback_details
.as_ref()
.and_then(|cb| cb.callback_id.clone()),
op_type: Some(operation_type_to_string(&op.r#type)),
sub_type: op.sub_type.clone(),
op_name: op.name.clone(),
due_at: operation_due_at(op),
parent_id: op.parent_id().map(str::to_owned),
start_timestamp: crate::to_system_time(*op.start_timestamp()),
end_timestamp: op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
};
(op.id().to_owned(), record)
}
fn operation_due_at(op: &Operation) -> Option<std::time::SystemTime> {
let timestamp = match op.r#type() {
OperationType::Wait => op
.wait_details
.as_ref()
.and_then(|d| d.scheduled_end_timestamp),
OperationType::Step => op
.step_details
.as_ref()
.and_then(|d| d.next_attempt_timestamp),
_ => None,
}?;
crate::to_system_time(timestamp)
}
fn operation_type_to_string(op_type: &OperationType) -> String {
op_type.as_str().to_owned()
}
pub(crate) fn execution_info_from_operation(
op: &Operation,
) -> Option<crate::plugin::OperationInfo> {
crate::plugin::OperationInfo::execution(
op.id(),
op.r#type().as_str(),
&sdk_status_to_checkpoint_status(op.status()),
op.parent_id().map(str::to_owned),
crate::to_system_time(*op.start_timestamp()),
op.end_timestamp().and_then(|t| crate::to_system_time(*t)),
)
}
#[cfg(test)]
pub(crate) fn operations_to_checkpoint_log(operations: &[Operation]) -> CheckpointLog {
let records: Vec<(String, CheckpointRecord)> = operations
.iter()
.filter(|op| op.r#type != OperationType::Execution)
.map(operation_to_record)
.collect();
CheckpointLog::from_records(records)
}
pub(crate) fn merge_operations_into_log(
log: &CheckpointLog,
operations: &[Operation],
) -> Vec<String> {
let mut changed = Vec::new();
for op in operations {
if op.r#type == OperationType::Execution {
continue;
}
let (wire_id, record) = operation_to_record(op);
let status = record.status.clone();
if log.insert(wire_id.clone(), record).as_ref() != Some(&status) {
changed.push(wire_id);
}
}
changed
}
#[derive(Debug)]
pub(crate) struct BootstrapState {
pub(crate) log: CheckpointLog,
pub(crate) input_payload: Option<String>,
pub(crate) execution: Option<crate::plugin::OperationInfo>,
}
pub(crate) async fn resolve_bootstrap_log(
client: &dyn ExecutionClient,
execution_arn: &str,
checkpoint_token: &str,
first_page: BootstrapState,
next_marker: Option<&str>,
) -> Result<BootstrapState, ExecutionClientError> {
let mut state = first_page;
let mut marker = next_marker.map(str::to_owned);
while let Some(current) = marker {
let page = client
.get_state(execution_arn, checkpoint_token, Some(¤t))
.await?;
marker = page.next_marker().map(str::to_owned);
if state.execution.is_none()
&& let Some(op) = page
.operations
.iter()
.find(|op| op.r#type == OperationType::Execution)
{
state.input_payload = op
.execution_details
.as_ref()
.and_then(|d| d.input_payload.clone());
state.execution = execution_info_from_operation(op);
}
for op in page
.operations
.iter()
.filter(|op| op.r#type != OperationType::Execution)
{
let (wire_id, record) = operation_to_record(op);
state.log.insert(wire_id, record);
}
}
Ok(state)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum PageInput {
NoExecutionOperation,
Execution(Option<String>),
}
impl PageInput {
pub(crate) fn into_payload(self) -> Option<String> {
match self {
Self::NoExecutionOperation => None,
Self::Execution(payload) => payload,
}
}
}
fn sdk_status_to_checkpoint_status(
status: &aws_sdk_lambda::types::OperationStatus,
) -> CheckpointStatus {
match status {
aws_sdk_lambda::types::OperationStatus::Started => CheckpointStatus::Started,
aws_sdk_lambda::types::OperationStatus::Pending => CheckpointStatus::Pending,
aws_sdk_lambda::types::OperationStatus::Ready => CheckpointStatus::Ready,
aws_sdk_lambda::types::OperationStatus::Succeeded => CheckpointStatus::Succeeded,
aws_sdk_lambda::types::OperationStatus::Failed => CheckpointStatus::Failed,
aws_sdk_lambda::types::OperationStatus::Cancelled => CheckpointStatus::Cancelled,
aws_sdk_lambda::types::OperationStatus::TimedOut => CheckpointStatus::TimedOut,
aws_sdk_lambda::types::OperationStatus::Stopped => CheckpointStatus::Stopped,
other => CheckpointStatus::Unknown(other.as_str().to_owned()),
}
}
fn extract_result(op: &Operation) -> Option<String> {
op.step_details
.as_ref()
.and_then(|s| s.result.clone())
.or_else(|| op.context_details.as_ref().and_then(|c| c.result.clone()))
.or_else(|| {
op.callback_details
.as_ref()
.and_then(|cb| cb.result.clone())
})
}
fn extract_error_type(op: &Operation) -> Option<String> {
op.step_details
.as_ref()
.and_then(|s| s.error.as_ref())
.and_then(|e| e.error_type.clone())
.or_else(|| {
op.context_details
.as_ref()
.and_then(|c| c.error.as_ref())
.and_then(|e| e.error_type.clone())
})
.or_else(|| {
op.callback_details
.as_ref()
.and_then(|cb| cb.error.as_ref())
.and_then(|e| e.error_type.clone())
})
}
fn extract_error_message(op: &Operation) -> Option<String> {
op.step_details
.as_ref()
.and_then(|s| s.error.as_ref())
.and_then(|e| e.error_message.clone())
.or_else(|| {
op.context_details
.as_ref()
.and_then(|c| c.error.as_ref())
.and_then(|e| e.error_message.clone())
})
.or_else(|| {
op.callback_details
.as_ref()
.and_then(|cb| cb.error.as_ref())
.and_then(|e| e.error_message.clone())
})
}
fn extract_wire_error_field<T>(
op: &Operation,
read: impl Fn(&aws_sdk_lambda::types::ErrorObject) -> Option<T>,
) -> Option<T> {
op.step_details
.as_ref()
.and_then(|s| s.error.as_ref())
.and_then(&read)
.or_else(|| {
op.context_details
.as_ref()
.and_then(|c| c.error.as_ref())
.and_then(&read)
})
.or_else(|| {
op.callback_details
.as_ref()
.and_then(|cb| cb.error.as_ref())
.and_then(&read)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sdk_operation_parent_and_timestamps() {
use std::time::{Duration, UNIX_EPOCH};
let with_all = Operation::builder()
.id("s")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.parent_id("p")
.start_timestamp(aws_smithy_types::DateTime::from_secs(10))
.end_timestamp(aws_smithy_types::DateTime::from_secs(12))
.build()
.expect("all required fields set");
let (_, record) = operation_to_record(&with_all);
assert_eq!(record.parent_id.as_deref(), Some("p"));
assert_eq!(
record.start_timestamp,
Some(UNIX_EPOCH + Duration::from_secs(10))
);
assert_eq!(
record.end_timestamp,
Some(UNIX_EPOCH + Duration::from_secs(12))
);
let bare = Operation::builder()
.id("s")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(10))
.build()
.expect("all required fields set");
let (_, record) = operation_to_record(&bare);
assert_eq!(record.parent_id, None);
assert_eq!(record.end_timestamp, None);
}
#[test]
fn unknown_type_and_status_keep_wire_tokens_on_get_state_path() {
let op = Operation::builder()
.id("f")
.r#type(OperationType::from("FUTURE_OP"))
.status(aws_sdk_lambda::types::OperationStatus::from(
"FUTURE_STATUS",
))
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.build()
.expect("all required fields set");
let (id, record) = operation_to_record(&op);
let info = crate::plugin::OperationInfo::from_record(&id, &record);
assert_eq!(info.kind(), None);
assert_eq!(info.status(), None);
assert_eq!(info.type_str(), Some("FUTURE_OP"));
assert_eq!(info.status_str(), Some("FUTURE_STATUS"));
}
#[test]
fn operation_type_to_string_returns_wire_tokens() {
for (op_type, token) in [
(OperationType::Callback, "CALLBACK"),
(OperationType::ChainedInvoke, "CHAINED_INVOKE"),
(OperationType::Context, "CONTEXT"),
(OperationType::Execution, "EXECUTION"),
(OperationType::Step, "STEP"),
(OperationType::Wait, "WAIT"),
] {
assert_eq!(operation_type_to_string(&op_type), token);
}
}
#[test]
fn execution_info_from_operation_reads_execution_only() {
use std::time::{Duration, UNIX_EPOCH};
let execution = Operation::builder()
.id("exec")
.r#type(OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(3))
.end_timestamp(aws_smithy_types::DateTime::from_secs(4))
.build()
.expect("all required fields set");
let info = execution_info_from_operation(&execution).expect("an EXECUTION operation");
assert_eq!(info.id(), "exec");
assert_eq!(info.type_str(), Some("EXECUTION"));
assert_eq!(info.status_str(), Some("STARTED"));
assert_eq!(info.parent_id(), None);
assert_eq!(
info.start_timestamp(),
Some(UNIX_EPOCH + Duration::from_secs(3))
);
assert_eq!(
info.end_timestamp(),
Some(UNIX_EPOCH + Duration::from_secs(4))
);
let step = Operation::builder()
.id("s")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(3))
.build()
.expect("all required fields set");
assert!(execution_info_from_operation(&step).is_none());
}
#[test]
fn due_at_from_wait_and_step_details() {
let op = |r#type: OperationType| {
Operation::builder()
.id("op")
.r#type(r#type)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
};
let wait = op(OperationType::Wait)
.wait_details(
aws_sdk_lambda::types::WaitDetails::builder()
.scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(100))
.build(),
)
.build()
.expect("all required fields set");
let step = op(OperationType::Step)
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.next_attempt_timestamp(aws_smithy_types::DateTime::from_secs(200))
.build(),
)
.build()
.expect("all required fields set");
let neither = op(OperationType::Wait)
.build()
.expect("all required fields set");
let callback = op(OperationType::Callback)
.wait_details(
aws_sdk_lambda::types::WaitDetails::builder()
.scheduled_end_timestamp(aws_smithy_types::DateTime::from_secs(300))
.build(),
)
.build()
.expect("all required fields set");
let epoch = std::time::UNIX_EPOCH;
assert_eq!(
operation_to_record(&wait).1.due_at,
Some(epoch + std::time::Duration::from_secs(100))
);
assert_eq!(
operation_to_record(&step).1.due_at,
Some(epoch + std::time::Duration::from_secs(200))
);
assert_eq!(operation_to_record(&neither).1.due_at, None);
assert_eq!(operation_to_record(&callback).1.due_at, None);
}
use aws_sdk_lambda::operation::checkpoint_durable_execution::{
CheckpointDurableExecutionError, CheckpointDurableExecutionOutput,
};
use aws_sdk_lambda::operation::get_durable_execution_state::{
GetDurableExecutionStateError, GetDurableExecutionStateOutput,
};
use aws_sdk_lambda::types::{CheckpointUpdatedExecutionState, OperationAction};
use aws_smithy_mocks::{RuleMode, mock, mock_client};
fn make_update(id: &str, action: OperationAction) -> OperationUpdate {
OperationUpdate::builder()
.id(id)
.r#type(OperationType::Step)
.action(action)
.build()
.expect("valid OperationUpdate")
}
#[tokio::test]
async fn checkpoint_maps_request_and_response() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.match_requests(|req| {
req.durable_execution_arn() == Some("arn:aws:lambda:us-east-1:1:function:f")
&& req.checkpoint_token() == Some("tok-in")
&& req.updates().len() == 2
&& req.updates().first().map(OperationUpdate::id) == Some("op-1")
&& req.updates().get(1).map(OperationUpdate::id) == Some("op-2")
})
.then_output(|| {
CheckpointDurableExecutionOutput::builder()
.checkpoint_token("tok-out")
.new_execution_state(
CheckpointUpdatedExecutionState::builder()
.operations(make_step_op("op-1", r#""r1""#))
.next_marker("more-pages")
.build(),
)
.build()
});
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let updates = vec![
make_update("op-1", OperationAction::Start),
make_update("op-2", OperationAction::Succeed),
];
let output = client
.checkpoint("arn:aws:lambda:us-east-1:1:function:f", "tok-in", updates)
.await
.expect("mocked checkpoint succeeds");
assert_eq!(output.checkpoint_token(), Some("tok-out"));
assert_eq!(output.updated_operations.len(), 1);
assert_eq!(
output.updated_operations.first().map(Operation::id),
Some("op-1")
);
assert_eq!(output.next_marker.as_deref(), Some("more-pages"));
assert_eq!(rule.num_calls(), 1);
}
const FIRST_ROOT_WIRE_ID: &str =
"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b";
async fn invoke_one_step(sdk_client: aws_sdk_lambda::Client) -> serde_json::Value {
let service = crate::wrap(
|_event: serde_json::Value, ctx: crate::DurableContext| async move {
let value: u32 = ctx.step(|_| async { Ok(7_u32) }).name("one").await?;
Ok::<u32, crate::BoxError>(value)
},
crate::Options::builder()
.execution_client(Arc::new(LambdaExecutionClient::new(sdk_client)))
.checkpoint_batching(false)
.build()
.expect("options build"),
);
let event = lambda_runtime::LambdaEvent::new(
crate::InvocationPayload(serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": {
"Operations": [
{
"Id": "root",
"Type": "Execution",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}
]
}
})),
lambda_runtime::Context::default(),
);
service(event)
.await
.map(|response| response.0)
.expect("a tokenless response answers PENDING, not an invocation error")
}
#[tokio::test]
async fn checkpoint_output_without_token_answers_pending_unless_execution_update() {
use std::sync::{Arc, Mutex as StdMutex};
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.then_output(|| CheckpointDurableExecutionOutput::builder().build());
let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
let output = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect("a tokenless response is Ok");
assert_eq!(output.checkpoint_token(), None);
let requests: Arc<StdMutex<Vec<Vec<OperationUpdate>>>> = Arc::default();
let seen = Arc::clone(&requests);
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.match_requests(move |req| {
seen.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(req.updates().to_vec());
true
})
.then_output(|| CheckpointDurableExecutionOutput::builder().build());
let response =
invoke_one_step(mock_client!(aws_sdk_lambda, RuleMode::MatchAny, [&rule])).await;
assert_eq!(response, serde_json::json!({ "Status": "PENDING" }));
let requests = requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
assert_eq!(requests.len(), 1, "{requests:?}");
let first = requests.first().expect("one request");
assert_eq!(first.len(), 1, "{first:?}");
let start = first.first().expect("the step's START");
assert_eq!(start.id(), FIRST_ROOT_WIRE_ID);
assert_eq!(start.action(), &OperationAction::Start);
assert!(
requests
.iter()
.flatten()
.all(|u| u.action() != &OperationAction::Fail),
"no terminal FAIL is sent: {requests:?}"
);
let (messages, _guard) = crate::driver::test_support::EventMessages::install();
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.then_output(|| CheckpointDurableExecutionOutput::builder().build());
let ctx = crate::DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::new(LambdaExecutionClient::new(mock_client!(
aws_sdk_lambda,
[&rule]
))),
"tok".to_owned(),
);
let succeed = OperationUpdate::builder()
.id("root")
.r#type(OperationType::Execution)
.action(OperationAction::Succeed)
.payload("1")
.build()
.expect("valid update");
ctx.checkpoint_updates_direct(vec![succeed])
.await
.expect("an EXECUTION update is exempt from withdrawal");
assert!(!ctx.coordinator().withdrawn());
assert_eq!(
ctx.coordinator().phase(),
crate::coordinator::Phase::Running
);
assert!(
messages.at(tracing::Level::WARN).is_empty(),
"{:?}",
messages.at(tracing::Level::WARN)
);
}
#[test]
fn execution_client_error_new_keeps_scope_and_message() {
use std::error::Error as _;
let err = ExecutionClientError::new(ExecutionClientErrorScope::Execution, "x");
assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
assert!(!err.is_retryable());
assert!(err.source().is_none());
assert_eq!(err.to_string(), "durable: client: x");
let err = ExecutionClientError::new(ExecutionClientErrorScope::Invocation, "x");
assert_eq!(err.scope(), ExecutionClientErrorScope::Invocation);
assert!(err.is_retryable());
}
#[tokio::test]
async fn lambda_transport_error_source_downcasts_to_sdk_error() {
use std::error::Error as _;
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.sequence()
.http_status(403, None)
.build();
let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
let err = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect_err("a 403 fails the call");
let source = err.source().expect("the transport keeps the SdkError");
let sdk = source
.downcast_ref::<CheckpointSdkError>()
.expect("the source is the final SdkError");
let text = format!("{}", aws_sdk_lambda::error::DisplayErrorContext(sdk));
assert_eq!(err.message(), text);
assert_eq!(format!("{err:#}"), format!("durable: client: {text}"));
assert_eq!(err.scope(), ExecutionClientErrorScope::Execution);
}
#[tokio::test]
async fn checkpoint_final_error_maps_to_non_retryable_client_error() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
CheckpointDurableExecutionError::InvalidParameterValueException(
aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
)
});
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect_err("modeled error must fail the call");
assert!(!err.is_retryable());
assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
}
#[tokio::test]
async fn checkpoint_stale_token_error_maps_to_retryable_client_error() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_error(|| {
CheckpointDurableExecutionError::InvalidParameterValueException(
aws_sdk_lambda::types::error::InvalidParameterValueException::builder()
.message("Invalid checkpoint token: token has been superseded")
.build(),
)
});
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect_err("stale checkpoint token must fail the call");
assert!(err.is_retryable());
assert_eq!(rule.num_calls(), 1, "a client fault is not retried");
}
#[tokio::test]
async fn checkpoint_retry_is_the_sdk_standard_retry_alone() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.sequence()
.http_status(503, None)
.repeatedly()
.build();
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect_err("persistent 503 must exhaust retries");
assert!(err.is_retryable());
assert_eq!(
rule.num_calls(),
3,
"attempts must equal the SDK standard-retry budget — no outer loop multiplying them"
);
}
#[tokio::test]
async fn get_state_returns_one_page_with_marker() {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
.match_requests(|req| {
req.durable_execution_arn() == Some("arn:test")
&& req.checkpoint_token() == Some("tok")
&& req.marker() == Some("m-1")
&& req.max_items() == Some(1000)
})
.then_output(|| {
GetDurableExecutionStateOutput::builder()
.operations(make_step_op("op-1", r#""p1""#))
.operations(make_step_op("op-2", r#""p2""#))
.next_marker("m-2")
.build()
.expect("operations is set")
});
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let page = client
.get_state("arn:test", "tok", Some("m-1"))
.await
.expect("get_state succeeds");
assert_eq!(rule.num_calls(), 1);
assert_eq!(page.operations.len(), 2);
assert_eq!(page.operations.first().map(Operation::id), Some("op-1"));
assert_eq!(page.operations.get(1).map(Operation::id), Some("op-2"));
assert_eq!(page.next_marker(), Some("m-2"));
}
struct ThreePageClient {
markers: Mutex<Vec<Option<String>>>,
}
impl ExecutionClient for ThreePageClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
self.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(marker.map(str::to_owned));
let page = match marker {
None => GetStateOutput::new(vec![make_step_op("a", "1")], Some("p-2".to_owned())),
Some("p-2") => {
GetStateOutput::new(vec![make_step_op("b", "2")], Some("p-3".to_owned()))
}
Some(_) => GetStateOutput::new(vec![make_step_op("c", "3")], None),
};
Box::pin(async move { Ok(page) })
}
}
#[tokio::test]
async fn sdk_loops_get_state_until_marker_empty() {
let client = ThreePageClient {
markers: Mutex::new(Vec::new()),
};
let operations = get_full_state(&client, "arn:test", "tok")
.await
.expect("whole state");
let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
assert_eq!(ids, ["a", "b", "c"]);
let markers = client
.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
assert_eq!(
markers,
[None, Some("p-2".to_owned()), Some("p-3".to_owned())]
);
}
struct InlineContinuationClient {
markers: Mutex<Vec<Option<String>>>,
}
impl ExecutionClient for InlineContinuationClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
self.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(marker.map(str::to_owned));
let page = match marker {
Some("m-1") => {
GetStateOutput::new(vec![make_step_op("s2", "2")], Some("m-2".to_owned()))
}
Some("m-2") => GetStateOutput::new(vec![make_step_op("s3", "3")], None),
_ => GetStateOutput::new(vec![make_step_op("s0", "0")], None),
};
Box::pin(async move { Ok(page) })
}
}
#[tokio::test]
async fn bootstrap_keeps_inline_page_and_continues_from_marker() {
let client = InlineContinuationClient {
markers: Mutex::new(Vec::new()),
};
let execution = Operation::builder()
.id("execution")
.r#type(OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
.execution_details(
aws_sdk_lambda::types::ExecutionDetails::builder()
.input_payload("{}")
.build(),
)
.build()
.expect("id, type, status and start timestamp are set");
let execution_info = execution_info_from_operation(&execution);
let first_page = BootstrapState {
log: operations_to_checkpoint_log(&[execution, make_step_op("s1", "1")]),
input_payload: Some("{}".to_owned()),
execution: execution_info,
};
let BootstrapState {
log,
input_payload: input,
..
} = resolve_bootstrap_log(&client, "arn:test", "tok", first_page, Some("m-1"))
.await
.expect("bootstrap succeeds");
let markers = client
.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
assert_eq!(markers, [Some("m-1".to_owned()), Some("m-2".to_owned())]);
for id in ["s1", "s2", "s3"] {
assert!(log.get(id).is_some(), "{id} must be in the merged log");
}
assert!(
log.get("s0").is_none(),
"the first page must not be re-read"
);
assert_eq!(input.as_deref(), Some("{}"));
}
fn make_execution_op(id: &str, input: Option<&str>) -> Operation {
let mut details = aws_sdk_lambda::types::ExecutionDetails::builder();
if let Some(input) = input {
details = details.input_payload(input);
}
Operation::builder()
.id(id)
.r#type(OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
.execution_details(details.build())
.build()
.expect("id, type, status and start timestamp are set")
}
struct MarkerPagesClient {
pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
markers: Mutex<Vec<Option<String>>>,
}
impl ExecutionClient for MarkerPagesClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
self.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(marker.map(str::to_owned));
let page = self
.pages
.iter()
.find(|(key, _, _)| Some(*key) == marker)
.map_or_else(
|| GetStateOutput::new(vec![], None),
|(_, ops, next)| GetStateOutput::new(ops.clone(), next.map(str::to_owned)),
);
Box::pin(async move { Ok(page) })
}
}
async fn resolve_input_from_pages(
pages: Vec<(&'static str, Vec<Operation>, Option<&'static str>)>,
) -> (Option<String>, Vec<Option<String>>) {
let client = MarkerPagesClient {
pages,
markers: Mutex::new(Vec::new()),
};
let BootstrapState {
input_payload: input,
..
} = resolve_bootstrap_log(
&client,
"arn:test",
"tok",
BootstrapState {
log: operations_to_checkpoint_log(&[]),
input_payload: None,
execution: None,
},
Some("m-1"),
)
.await
.expect("bootstrap succeeds");
let markers = client
.markers
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
(input, markers)
}
#[tokio::test]
async fn bootstrap_input_comes_from_first_fetched_execution_operation() {
let (input, markers) = resolve_input_from_pages(vec![
("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
(
"m-2",
vec![make_execution_op("execution", Some(r#"{"count":1}"#))],
Some("m-3"),
),
(
"m-3",
vec![make_execution_op("late", Some(r#"{"late":1}"#))],
None,
),
])
.await;
assert_eq!(input.as_deref(), Some(r#"{"count":1}"#));
assert_eq!(
markers,
[
Some("m-1".to_owned()),
Some("m-2".to_owned()),
Some("m-3".to_owned())
]
);
let (input, markers) = resolve_input_from_pages(vec![
("m-1", vec![make_step_op("s1", "1")], Some("m-2")),
(
"m-2",
vec![make_execution_op("execution", None)],
Some("m-3"),
),
(
"m-3",
vec![make_execution_op("late", Some(r#"{"late":1}"#))],
None,
),
])
.await;
assert_eq!(input, None, "an absent first payload must not be replaced");
assert_eq!(markers.len(), 3, "pagination still reaches the last page");
}
#[test]
fn output_accessors_read_empty_values_as_none() {
let output = CheckpointOutput::new(Some(String::new()), vec![], None);
assert_eq!(output.checkpoint_token(), None);
let page = GetStateOutput::new(vec![], Some(String::new()));
assert_eq!(page.next_marker(), None);
}
struct EmptyMarkerClient {
calls: Mutex<u32>,
}
impl ExecutionClient for EmptyMarkerClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(CheckpointOutput::new(Some("t".to_owned()), vec![], None)) })
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
*self
.calls
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += 1;
let page = GetStateOutput::new(vec![make_step_op("a", "1")], Some(String::new()));
Box::pin(async move { Ok(page) })
}
}
#[tokio::test]
async fn sdk_stops_paginating_on_empty_marker() {
let client = EmptyMarkerClient {
calls: Mutex::new(0),
};
let operations = get_full_state(&client, "arn:test", "tok")
.await
.expect("whole state");
let ids: Vec<&str> = operations.iter().map(Operation::id).collect();
assert_eq!(ids, ["a"]);
assert_eq!(
*client
.calls
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
1
);
}
#[tokio::test]
async fn checkpoint_maps_empty_token_and_marker_to_none() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_output(|| {
CheckpointDurableExecutionOutput::builder()
.checkpoint_token("")
.new_execution_state(
CheckpointUpdatedExecutionState::builder()
.operations(make_step_op("op-1", r#""r1""#))
.next_marker("")
.build(),
)
.build()
});
let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
let output = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect("mocked checkpoint succeeds");
assert_eq!(output.checkpoint_token, None);
assert_eq!(output.next_marker, None);
assert_eq!(output.updated_operations.len(), 1);
}
#[tokio::test]
async fn get_state_maps_empty_marker_to_none() {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_output(|| {
GetDurableExecutionStateOutput::builder()
.operations(make_step_op("op-1", r#""p1""#))
.next_marker("")
.build()
.expect("operations is set")
});
let client = LambdaExecutionClient::new(mock_client!(aws_sdk_lambda, [&rule]));
let page = client
.get_state("arn:test", "tok", None)
.await
.expect("get_state succeeds");
assert_eq!(page.next_marker, None);
assert_eq!(page.operations.len(), 1);
}
#[tokio::test]
async fn get_state_final_error_maps_to_retryable_client_error() {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(|| {
GetDurableExecutionStateError::InvalidParameterValueException(
aws_sdk_lambda::types::error::InvalidParameterValueException::builder().build(),
)
});
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.get_state("arn:test", "tok", None)
.await
.expect_err("modeled error must fail the call");
assert!(err.is_retryable());
}
#[tokio::test]
async fn checkpoint_unmodeled_403_fails_the_execution() {
let rule = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.sequence()
.http_status(403, None)
.repeatedly()
.build();
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect_err("a 403 must fail the call");
assert!(!err.is_retryable());
assert_eq!(rule.num_calls(), 1, "a 403 is not retried");
}
#[tokio::test]
async fn get_state_kms_errors_classify_non_retryable() {
use aws_sdk_lambda::types::error::{
KmsAccessDeniedException, KmsDisabledException, KmsInvalidStateException,
KmsNotFoundException,
};
type Make = fn() -> GetDurableExecutionStateError;
let cases: [(Make, &str); 4] = [
(
|| {
GetDurableExecutionStateError::KmsAccessDeniedException(
KmsAccessDeniedException::builder().build(),
)
},
"KMSAccessDeniedException",
),
(
|| {
GetDurableExecutionStateError::KmsDisabledException(
KmsDisabledException::builder().build(),
)
},
"KMSDisabledException",
),
(
|| {
GetDurableExecutionStateError::KmsInvalidStateException(
KmsInvalidStateException::builder().build(),
)
},
"KMSInvalidStateException",
),
(
|| {
GetDurableExecutionStateError::KmsNotFoundException(
KmsNotFoundException::builder().build(),
)
},
"KMSNotFoundException",
),
];
for (make, wire_name) in cases {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_error(make);
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.get_state("arn:test", "tok", None)
.await
.expect_err("a KMS error must fail the call");
assert!(!err.is_retryable(), "{wire_name} must be non-retryable");
assert!(
err.to_string().contains(wire_name),
"message must name {wire_name}, got: {err}"
);
}
}
#[tokio::test]
async fn get_state_plain_4xx_stays_retryable() {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
.sequence()
.http_status(403, None)
.repeatedly()
.build();
let sdk_client = mock_client!(aws_sdk_lambda, [&rule]);
let client = LambdaExecutionClient::new(sdk_client);
let err = client
.get_state("arn:test", "tok", None)
.await
.expect_err("a 403 must fail the call");
assert!(err.is_retryable());
}
#[tokio::test]
async fn accept_then_retryable_error_records_then_fails() {
let client = InMemoryExecutionClient::new(Vec::new());
client.enqueue_checkpoint_response(TestResponse::AcceptThenRetryableError(
"lost response".to_owned(),
));
let update = OperationUpdate::builder()
.id("e")
.r#type(OperationType::Execution)
.action(OperationAction::Succeed)
.build()
.expect("valid update");
let err = client
.checkpoint("arn", "token", vec![update])
.await
.expect_err("a retryable error");
assert!(err.is_retryable());
assert_eq!(err.message(), "lost response");
assert_eq!(client.recorded_updates().len(), 1);
}
#[tokio::test]
async fn in_memory_client_returns_queued_responses_in_order() {
let client = InMemoryExecutionClient::new(Vec::new());
client.enqueue_checkpoint_response(TestResponse::RetryableError("throttled".to_owned()));
client.enqueue_checkpoint_response(TestResponse::NonRetryableError("invalid".to_owned()));
client.enqueue_checkpoint_response(TestResponse::Success(Vec::new()));
let first = client.checkpoint("arn:test", "token-0", Vec::new()).await;
let first_err = first.unwrap_err();
assert!(first_err.is_retryable());
let second = client.checkpoint("arn:test", "token-0", Vec::new()).await;
let second_err = second.unwrap_err();
assert!(!second_err.is_retryable());
let third = client.checkpoint("arn:test", "token-0", Vec::new()).await;
assert!(third.is_ok());
let count = *client
.checkpoint_call_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(count, 3);
}
#[tokio::test]
async fn success_returns_checkpoint_output() {
let client = InMemoryExecutionClient::new(Vec::new());
let result = client.checkpoint("arn:test", "token-0", Vec::new()).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(
output
.checkpoint_token()
.is_some_and(|t| t.starts_with("token-"))
);
}
#[test]
fn checkpoint_log_from_operations_maps_statuses() {
let ops = vec![
Operation::builder()
.id("op-1")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.attempt(1)
.result(r#""hello""#)
.build(),
)
.build()
.unwrap(),
Operation::builder()
.id("op-2")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Failed)
.start_timestamp(aws_smithy_types::DateTime::from_secs(1))
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.attempt(2)
.error(
aws_sdk_lambda::types::ErrorObject::builder()
.error_type("StepError")
.error_message("oops")
.build(),
)
.build(),
)
.build()
.unwrap(),
];
let log = operations_to_checkpoint_log(&ops);
let r1 = log.get("op-1");
assert!(r1.is_some());
let r1 = r1.unwrap();
assert_eq!(r1.status, CheckpointStatus::Succeeded);
assert_eq!(r1.result.as_deref(), Some(r#""hello""#));
assert_eq!(r1.attempt, 1);
let r2 = log.get("op-2");
assert!(r2.is_some());
let r2 = r2.unwrap();
assert_eq!(r2.status, CheckpointStatus::Failed);
assert_eq!(r2.error_type.as_deref(), Some("StepError"));
assert_eq!(r2.error_message.as_deref(), Some("oops"));
assert_eq!(r2.attempt, 2);
}
#[test]
fn chained_invoke_maps_to_invoke_specific_fields() {
let ops = vec![
Operation::builder()
.id("invoke-ok")
.r#type(OperationType::ChainedInvoke)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.chained_invoke_details(
aws_sdk_lambda::types::ChainedInvokeDetails::builder()
.result(r#""invoke-payload""#)
.build(),
)
.build()
.unwrap(),
Operation::builder()
.id("invoke-err")
.r#type(OperationType::ChainedInvoke)
.status(aws_sdk_lambda::types::OperationStatus::Failed)
.start_timestamp(aws_smithy_types::DateTime::from_secs(1))
.chained_invoke_details(
aws_sdk_lambda::types::ChainedInvokeDetails::builder()
.error(
aws_sdk_lambda::types::ErrorObject::builder()
.error_type("InvokeError")
.error_message("downstream failed")
.build(),
)
.build(),
)
.build()
.unwrap(),
];
let log = operations_to_checkpoint_log(&ops);
let ok = log.get("invoke-ok");
assert!(ok.is_some());
let ok = ok.unwrap();
assert_eq!(ok.invoke_result.as_deref(), Some(r#""invoke-payload""#));
assert_eq!(
ok.result, None,
"invoke results must not leak into the generic result field"
);
let err = log.get("invoke-err");
assert!(err.is_some());
let err = err.unwrap();
assert_eq!(err.invoke_error_type.as_deref(), Some("InvokeError"));
assert_eq!(
err.invoke_error_message.as_deref(),
Some("downstream failed")
);
assert_eq!(
err.error_type, None,
"invoke errors must not leak into the generic error fields"
);
assert_eq!(err.error_message, None);
}
#[test]
fn context_replay_children_is_preserved() {
let ops = vec![
Operation::builder()
.id("child-replay")
.r#type(OperationType::Context)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.context_details(
aws_sdk_lambda::types::ContextDetails::builder()
.replay_children(true)
.build(),
)
.build()
.unwrap(),
Operation::builder()
.id("child-inline")
.r#type(OperationType::Context)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(1))
.context_details(
aws_sdk_lambda::types::ContextDetails::builder()
.result(r#""child-result""#)
.build(),
)
.build()
.unwrap(),
];
let log = operations_to_checkpoint_log(&ops);
let replayed = log.get("child-replay");
assert!(replayed.is_some());
let replayed = replayed.unwrap();
assert!(
replayed.replay_children,
"ReplayChildren must survive the typed-state conversion"
);
let inline = log.get("child-inline");
assert!(inline.is_some());
let inline = inline.unwrap();
assert!(!inline.replay_children);
assert_eq!(inline.result.as_deref(), Some(r#""child-result""#));
}
#[test]
fn execution_operations_are_filtered_from_log() {
let ops = vec![
Operation::builder()
.id("exec-0")
.r#type(OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.build()
.unwrap(),
Operation::builder()
.id("step-1")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(1))
.build()
.unwrap(),
];
let log = operations_to_checkpoint_log(&ops);
assert!(
log.get("exec-0").is_none(),
"Execution operations must be filtered like the JSON path does"
);
assert!(log.get("step-1").is_some());
}
#[tokio::test]
async fn get_state_returns_preloaded_operations() {
let ops = vec![
Operation::builder()
.id("op-1")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.build()
.unwrap(),
];
let client = InMemoryExecutionClient::new(ops);
let result = client.get_state("arn:test", "token-0", None).await;
assert!(result.is_ok());
let output = result.unwrap();
assert_eq!(output.operations.len(), 1);
assert_eq!(output.operations.first().map(Operation::id), Some("op-1"));
}
#[test]
fn merge_operations_inserts_callback_id_into_existing_log() {
let log = CheckpointLog::from_records(vec![(
"op-cb".to_owned(),
CheckpointRecord {
id: "op-cb".to_owned(),
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)]);
let updated_ops = vec![
Operation::builder()
.id("op-cb")
.r#type(OperationType::Callback)
.status(aws_sdk_lambda::types::OperationStatus::Pending)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.callback_details(
aws_sdk_lambda::types::CallbackDetails::builder()
.callback_id("assigned-cb-id-42")
.build(),
)
.build()
.unwrap(),
];
merge_operations_into_log(&log, &updated_ops);
let record = log.get("op-cb");
assert!(record.is_some());
let record = record.unwrap();
assert_eq!(record.callback_id.as_deref(), Some("assigned-cb-id-42"));
assert_eq!(record.status, CheckpointStatus::Pending);
}
#[test]
fn merge_operations_adds_new_record_to_log() {
let log = CheckpointLog::empty();
let updated_ops = vec![
Operation::builder()
.id("new-op")
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.result(r#""merged result""#)
.build(),
)
.build()
.unwrap(),
];
merge_operations_into_log(&log, &updated_ops);
let record = log.get("new-op");
assert!(record.is_some());
let record = record.unwrap();
assert_eq!(record.status, CheckpointStatus::Succeeded);
assert_eq!(record.result.as_deref(), Some(r#""merged result""#));
}
fn status_op(
id: &str,
ty: OperationType,
status: aws_sdk_lambda::types::OperationStatus,
callback_id: Option<&str>,
) -> Operation {
let mut builder = Operation::builder()
.id(id)
.r#type(ty)
.status(status)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0));
if let Some(callback_id) = callback_id {
builder = builder.callback_details(
aws_sdk_lambda::types::CallbackDetails::builder()
.callback_id(callback_id)
.build(),
);
}
builder.build().unwrap()
}
#[test]
fn merge_reports_status_changes() {
use aws_sdk_lambda::types::OperationStatus as S;
let log = CheckpointLog::from_records(vec![
(
"b".to_owned(),
CheckpointRecord::with_status("b", CheckpointStatus::Started),
),
(
"c".to_owned(),
CheckpointRecord::with_status("c", CheckpointStatus::Started),
),
]);
let changed = merge_operations_into_log(
&log,
&[
status_op("a", OperationType::Step, S::Started, None),
status_op("b", OperationType::Callback, S::Started, Some("cb-1")),
status_op("c", OperationType::Step, S::Succeeded, None),
status_op("exec", OperationType::Execution, S::Started, None),
],
);
assert_eq!(changed, ["a", "c"]);
assert_eq!(
log.get("b").unwrap().callback_id.as_deref(),
Some("cb-1"),
"the field-only change is still merged"
);
assert!(!log.contains("exec"));
}
#[test]
fn merge_compares_unknown_statuses_by_token() {
use aws_sdk_lambda::types::OperationStatus as S;
let log = CheckpointLog::from_records(vec![(
"u".to_owned(),
CheckpointRecord::with_status("u", CheckpointStatus::Unknown("A".to_owned())),
)]);
let op = |token: &str| status_op("u", OperationType::Step, S::from(token), None);
assert_eq!(
merge_operations_into_log(&log, &[op("A")]),
Vec::<String>::new()
);
assert_eq!(merge_operations_into_log(&log, &[op("B")]), ["u"]);
}
fn make_step_op(id: &str, result: &str) -> Operation {
Operation::builder()
.id(id)
.r#type(OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.result(result)
.build(),
)
.build()
.unwrap()
}
#[tokio::test]
async fn in_memory_get_state_returns_all_operations() {
let ops = vec![
make_step_op("step-1", "\"page1\""),
make_step_op("step-2", "\"page2\""),
make_step_op("step-3", "\"page3\""),
];
let client = InMemoryExecutionClient::new(ops);
let state = client.get_state("arn:test", "token-0", None).await;
assert!(state.is_ok());
let state = state.unwrap();
assert_eq!(state.operations.len(), 3);
}
#[tokio::test]
async fn checkpoint_with_marker_returns_paginated_response() {
let all_ops = vec![
make_step_op("step-1", "\"r1\""),
make_step_op("step-2", "\"r2\""),
make_step_op("step-3", "\"r3\""),
];
let client = InMemoryExecutionClient::new(all_ops);
let page1_ops = vec![make_step_op("step-1", "\"r1\"")];
client.enqueue_checkpoint_response(TestResponse::SuccessPaginated(
page1_ops,
"marker-page-2".to_owned(),
));
let result = client.checkpoint("arn:test", "token-0", vec![]).await;
assert!(result.is_ok());
let output = result.unwrap();
assert_eq!(output.updated_operations.len(), 1);
assert_eq!(output.next_marker, Some("marker-page-2".to_owned()));
let full_state = client
.get_state(
"arn:test",
output.checkpoint_token().unwrap_or_default(),
None,
)
.await;
assert!(full_state.is_ok());
let full_state = full_state.unwrap();
assert_eq!(full_state.operations.len(), 3);
}
#[test]
fn operations_to_checkpoint_log_handles_multi_page_operations() {
let ops = vec![
make_step_op("step-1", "\"result-1\""),
make_step_op("step-2", "\"result-2\""),
make_step_op("step-3", "\"result-3\""),
];
let log = operations_to_checkpoint_log(&ops);
assert!(log.get("step-1").is_some());
assert!(log.get("step-2").is_some());
assert!(log.get("step-3").is_some());
let r1 = log.get("step-1").unwrap();
assert_eq!(r1.result.as_deref(), Some("\"result-1\""));
assert_eq!(r1.status, CheckpointStatus::Succeeded);
let r3 = log.get("step-3").unwrap();
assert_eq!(r3.result.as_deref(), Some("\"result-3\""));
}
use aws_sdk_lambda::config::http::HttpResponse;
use aws_sdk_lambda::error::SdkError;
use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionError as CkptErr;
use aws_smithy_runtime_api::http::StatusCode;
use aws_smithy_types::body::SdkBody;
fn raw_response(status: u16) -> HttpResponse {
HttpResponse::new(
StatusCode::try_from(status).expect("valid HTTP status"),
SdkBody::empty(),
)
}
fn classify<E: Into<CkptErr>>(err: E, status: u16) -> ExecutionClientError {
classify_checkpoint_error(SdkError::service_error(err.into(), raw_response(status)))
}
fn invalid_parameter(message: Option<&str>) -> CkptErr {
let mut builder = aws_sdk_lambda::types::error::InvalidParameterValueException::builder();
if let Some(m) = message {
builder = builder.message(m);
}
CkptErr::InvalidParameterValueException(builder.build())
}
#[test]
fn throttling_and_5xx_classify_retryable() {
let throttled = classify(
CkptErr::TooManyRequestsException(
aws_sdk_lambda::types::error::TooManyRequestsException::builder().build(),
),
429,
);
assert!(throttled.is_retryable());
let server = classify(
CkptErr::ServiceException(
aws_sdk_lambda::types::error::ServiceException::builder().build(),
),
500,
);
assert!(server.is_retryable());
}
#[test]
fn invalid_parameter_classifies_non_retryable() {
assert!(!classify(invalid_parameter(Some("payload too large")), 400).is_retryable());
assert!(!classify(invalid_parameter(None), 400).is_retryable());
}
#[test]
fn stale_token_rejection_classifies_retryable() {
let stale = classify(
invalid_parameter(Some("Invalid checkpoint token: token has been superseded")),
400,
);
assert!(stale.is_retryable());
let not_prefix = classify(
invalid_parameter(Some(
"field X rejected (not an Invalid checkpoint token case)",
)),
400,
);
assert!(!not_prefix.is_retryable());
let wrong_case = classify(
invalid_parameter(Some("Invalid Checkpoint Token: old non-service spelling")),
400,
);
assert!(!wrong_case.is_retryable());
}
#[test]
fn kms_misconfiguration_classifies_non_retryable() {
let kms = classify(
CkptErr::KmsAccessDeniedException(
aws_sdk_lambda::types::error::KmsAccessDeniedException::builder().build(),
),
502,
);
assert!(!kms.is_retryable());
}
#[test]
fn unhandled_4xx_classifies_non_retryable() {
for status in [400, 403, 404, 409, 413] {
let err = classify(CkptErr::unhandled("unmodeled rejection"), status);
assert!(!err.is_retryable(), "status {status} must be non-retryable");
}
}
#[test]
fn unhandled_429_classifies_retryable() {
assert!(classify(CkptErr::unhandled("throttled"), 429).is_retryable());
}
#[test]
fn unhandled_5xx_classifies_retryable() {
for status in [500, 502, 503] {
let err = classify(CkptErr::unhandled("some new service error"), status);
assert!(err.is_retryable(), "status {status} must be retryable");
}
}
#[test]
fn response_error_4xx_classifies_non_retryable() {
let err: SdkError<CkptErr, HttpResponse> =
SdkError::response_error("unparseable body", raw_response(403));
assert!(!classify_checkpoint_error(err).is_retryable());
}
#[test]
fn response_error_200_classifies_retryable() {
let err: SdkError<CkptErr, HttpResponse> =
SdkError::response_error("unparseable body", raw_response(200));
assert!(classify_checkpoint_error(err).is_retryable());
}
#[test]
fn transport_failures_classify_retryable() {
let timeout: SdkError<CkptErr, HttpResponse> = SdkError::timeout_error("connect timed out");
assert!(classify_checkpoint_error(timeout).is_retryable());
}
#[test]
fn checkpoint_error_variant_canary() {
const CLASSIFIED_VARIANTS: &[&str] = &[
"InvalidParameterValueException",
"KmsAccessDeniedException",
"KmsDisabledException",
"KmsInvalidStateException",
"KmsNotFoundException",
"ServiceException",
"TooManyRequestsException",
"Unhandled",
];
let source = read_sdk_source("src/operation/checkpoint_durable_execution.rs");
let mut actual = extract_enum_variants(&source, "CheckpointDurableExecutionError");
actual.sort_unstable();
let mut expected: Vec<String> = CLASSIFIED_VARIANTS
.iter()
.map(|s| (*s).to_owned())
.collect();
expected.sort_unstable();
assert_eq!(
actual, expected,
"aws-sdk-lambda's CheckpointDurableExecutionError variant set changed. \
Reclassify the new/removed variants in classify_checkpoint_error \
(src/client.rs), then update this canary's pinned list."
);
}
#[test]
#[expect(clippy::too_many_lines)] fn upstream_type_enum_variant_canary() {
struct Pin {
file: &'static str,
enum_name: &'static str,
variants: &'static [&'static str],
wildcard_sites: &'static str,
}
let pins = [
Pin {
file: "src/types/_operation_status.rs",
enum_name: "OperationStatus",
variants: &[
"Cancelled",
"Failed",
"Pending",
"Ready",
"Started",
"Stopped",
"Succeeded",
"TimedOut",
"Unknown",
],
wildcard_sites: "sdk_status_to_checkpoint_status (src/client.rs); \
operation_status_str and operation_status_wire_str \
(src/test_util.rs)",
},
Pin {
file: "src/types/_operation_type.rs",
enum_name: "OperationType",
variants: &[
"Callback",
"ChainedInvoke",
"Context",
"Execution",
"Step",
"Unknown",
"Wait",
],
wildcard_sites: "operation_type_str (src/tracing_layer.rs); \
operation_type_str (src/test_util.rs)",
},
Pin {
file: "src/types/_operation_action.rs",
enum_name: "OperationAction",
variants: &["Cancel", "Fail", "Retry", "Start", "Succeed", "Unknown"],
wildcard_sites: "PendingTransitionEvent::emit (src/tracing_layer.rs); \
BackendState::apply_update (src/test_util.rs); \
child_starts_before_first_terminal \
(src/map_parallel.rs tests)",
},
Pin {
file: "src/types/_execution_status.rs",
enum_name: "ExecutionStatus",
variants: &[
"Failed",
"Running",
"Stopped",
"Succeeded",
"TimedOut",
"Unknown",
],
wildcard_sites: "execution_status_str (src/test_util.rs)",
},
Pin {
file: "src/types/_event_type.rs",
enum_name: "EventType",
variants: &[
"CallbackFailed",
"CallbackStarted",
"CallbackSucceeded",
"CallbackTimedOut",
"ChainedInvokeFailed",
"ChainedInvokeStarted",
"ChainedInvokeStopped",
"ChainedInvokeSucceeded",
"ChainedInvokeTimedOut",
"ContextFailed",
"ContextStarted",
"ContextSucceeded",
"ExecutionFailed",
"ExecutionStarted",
"ExecutionStopped",
"ExecutionSucceeded",
"ExecutionTimedOut",
"InvocationCompleted",
"StepFailed",
"StepStarted",
"StepSucceeded",
"Unknown",
"WaitCancelled",
"WaitStarted",
"WaitSucceeded",
],
wildcard_sites: "apply_event_to_op and event_op_type (src/test_util.rs)",
},
];
for pin in pins {
let source = read_sdk_source(pin.file);
let mut actual = extract_enum_variants(&source, pin.enum_name);
actual.sort_unstable();
let mut expected: Vec<String> = pin.variants.iter().map(|s| (*s).to_owned()).collect();
expected.sort_unstable();
assert_eq!(
actual, expected,
"aws-sdk-lambda's {} variant set changed. Classify the new/removed \
variants explicitly at every wildcard match site — {} — then update \
this canary's pinned list.",
pin.enum_name, pin.wildcard_sites
);
}
}
#[test]
fn unknown_operation_status_maps_to_unknown_carrying_raw() {
use aws_sdk_lambda::types::OperationStatus;
let novel = OperationStatus::from("PAUSED");
assert_eq!(
sdk_status_to_checkpoint_status(&novel),
CheckpointStatus::Unknown("PAUSED".to_owned())
);
let modeled = [
(OperationStatus::Started, CheckpointStatus::Started),
(OperationStatus::Pending, CheckpointStatus::Pending),
(OperationStatus::Ready, CheckpointStatus::Ready),
(OperationStatus::Succeeded, CheckpointStatus::Succeeded),
(OperationStatus::Failed, CheckpointStatus::Failed),
(OperationStatus::Cancelled, CheckpointStatus::Cancelled),
(OperationStatus::TimedOut, CheckpointStatus::TimedOut),
(OperationStatus::Stopped, CheckpointStatus::Stopped),
];
for (sdk, expected) in modeled {
assert_eq!(
sdk_status_to_checkpoint_status(&sdk),
expected,
"modeled variant {sdk:?} must map explicitly"
);
}
}
fn read_sdk_source(relative: &str) -> String {
let manifest_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let lockfile = std::fs::read_to_string(manifest_dir.join("Cargo.lock"))
.expect("Cargo.lock readable at the workspace root");
let version = lockfile
.split("[[package]]")
.find(|block| block.contains("name = \"aws-sdk-lambda\""))
.and_then(|block| {
block
.lines()
.find_map(|line| line.trim().strip_prefix("version = \""))
.map(|rest| rest.trim_end_matches('"').to_owned())
})
.expect("aws-sdk-lambda pinned in Cargo.lock");
let cargo_home = std::env::var_os("CARGO_HOME").map_or_else(
|| {
let home = std::env::var_os("HOME").expect("HOME set");
std::path::PathBuf::from(home).join(".cargo")
},
std::path::PathBuf::from,
);
let registry_src = cargo_home.join("registry").join("src");
let crate_dir_name = format!("aws-sdk-lambda-{version}");
let crate_dir = std::fs::read_dir(®istry_src)
.expect("cargo registry src directory readable")
.filter_map(Result::ok)
.map(|entry| entry.path().join(&crate_dir_name))
.find(|candidate| candidate.is_dir())
.expect("pinned aws-sdk-lambda version extracted in the cargo registry");
std::fs::read_to_string(crate_dir.join(relative)).expect("SDK source file readable")
}
#[expect(clippy::panic, clippy::map_unwrap_or)]
fn extract_enum_variants(source: &str, enum_name: &str) -> Vec<String> {
let header = format!("pub enum {enum_name} {{");
let start = source
.find(&header)
.unwrap_or_else(|| panic!("enum {enum_name} present in SDK source"));
let body_start = start + header.len();
let body_end = source[body_start..]
.find("\n}")
.map(|offset| body_start + offset)
.unwrap_or_else(|| panic!("enum {enum_name} body terminated"));
let body = &source[body_start..body_end];
body.lines()
.filter_map(|line| {
let trimmed = line.trim();
let first = trimmed.chars().next()?;
if !first.is_ascii_uppercase() {
return None;
}
let ident: String = trimmed
.chars()
.take_while(char::is_ascii_alphanumeric)
.collect();
let rest = trimmed.get(ident.len()..)?;
if ident.is_empty() || !(rest.starts_with('(') || rest.starts_with(',')) {
return None;
}
Some(ident)
})
.collect()
}
use aws_sdk_lambda::config::{AppName, BehaviorVersion, Credentials, Region};
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
use aws_smithy_types::retry::RetryConfig;
fn expected_token() -> String {
format!(
"aws-durable-execution-sdk-rust/{}",
env!("CARGO_PKG_VERSION")
)
}
fn replay_event(status: u16, body: &'static str) -> ReplayEvent {
let response = HttpResponse::new(
StatusCode::try_from(status).expect("valid status"),
SdkBody::from(body),
);
ReplayEvent::new(HttpRequest::new(SdkBody::empty()), response)
}
fn replay_lambda_client(
http: &StaticReplayClient,
app_name: Option<AppName>,
) -> LambdaExecutionClient {
let mut config = aws_sdk_lambda::config::Builder::new()
.behavior_version(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.credentials_provider(Credentials::new("akid", "secret", None, None, "test"))
.retry_config(
RetryConfig::standard().with_initial_backoff(std::time::Duration::from_millis(1)),
)
.http_client(http.clone());
config.set_app_name(app_name);
LambdaExecutionClient::new(aws_sdk_lambda::Client::from_conf(config.build()))
}
fn user_agent_headers(http: &StaticReplayClient) -> Vec<Vec<String>> {
http.actual_requests()
.map(|req| {
req.headers()
.get_all("user-agent")
.map(str::to_owned)
.collect()
})
.collect()
}
fn assert_one_token(headers: &[String]) {
assert_eq!(
headers.len(),
1,
"exactly one User-Agent header: {headers:?}"
);
let value = headers.first().expect("one header");
let token = expected_token();
let count = value.split(' ').filter(|part| *part == token).count();
assert_eq!(count, 1, "exactly one {token} part in {value:?}");
}
#[tokio::test]
async fn user_agent_token_on_checkpoint() {
let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
let client = replay_lambda_client(&http, None);
let output = client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect("replayed checkpoint succeeds");
assert_eq!(output.checkpoint_token(), Some("t-1"));
let attempts = user_agent_headers(&http);
assert_eq!(attempts.len(), 1);
for headers in &attempts {
assert_one_token(headers);
}
}
#[tokio::test]
async fn user_agent_token_on_get_state() {
let http = StaticReplayClient::new(vec![replay_event(200, r#"{"Operations":[]}"#)]);
let client = replay_lambda_client(&http, None);
let output = client
.get_state("arn:test", "tok", None)
.await
.expect("replayed state read succeeds");
assert_eq!(output.next_marker(), None);
let attempts = user_agent_headers(&http);
assert_eq!(attempts.len(), 1, "one get_state call sends one request");
for headers in &attempts {
assert_one_token(headers);
}
}
#[tokio::test]
async fn user_agent_token_once_per_retry_attempt() {
let http = StaticReplayClient::new(vec![
replay_event(503, "{}"),
replay_event(200, r#"{"CheckpointToken":"t-1"}"#),
]);
let client = replay_lambda_client(&http, None);
client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect("the retry succeeds");
let attempts = user_agent_headers(&http);
assert_eq!(attempts.len(), 2, "the 503 is retried once");
for headers in &attempts {
assert_one_token(headers);
}
}
#[tokio::test]
async fn user_agent_keeps_app_name() {
let http = StaticReplayClient::new(vec![replay_event(200, r#"{"CheckpointToken":"t-1"}"#)]);
let app_name = AppName::new("my-app").expect("valid app name");
let client = replay_lambda_client(&http, Some(app_name));
client
.checkpoint("arn:test", "tok", Vec::new())
.await
.expect("replayed checkpoint succeeds");
let token = expected_token();
let requests: Vec<_> = http.actual_requests().collect();
assert_eq!(requests.len(), 1);
for req in requests {
let user_agent = req.headers().get("user-agent").expect("User-Agent header");
assert!(
user_agent.split(' ').any(|part| part == "app/my-app"),
"the app name stays in {user_agent:?}"
);
let amz_user_agent = req
.headers()
.get("x-amz-user-agent")
.expect("x-amz-user-agent header");
assert!(
!amz_user_agent.contains(&token),
"x-amz-user-agent stays unchanged: {amz_user_agent:?}"
);
}
}
#[test]
fn user_agent_with_token_appends_or_starts() {
let token = expected_token();
assert_eq!(
user_agent_with_token(Some("aws-sdk-rust/1.3.0 os/linux")),
format!("aws-sdk-rust/1.3.0 os/linux {token}")
);
assert_eq!(user_agent_with_token(None), token);
}
fn is_version_number(s: &str) -> bool {
!s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) && (s == "0" || !s.starts_with('0'))
}
fn has_supported_version_form(version: &str) -> bool {
let (core, pre_release) = match version.split_once('-') {
Some((core, pre_release)) => (core, Some(pre_release)),
None => (version, None),
};
let numbers: Vec<&str> = core.split('.').collect();
if numbers.len() != 3 || !numbers.iter().all(|n| is_version_number(n)) {
return false;
}
let Some(pre_release) = pre_release else {
return true;
};
let (label, number) = match pre_release.split_once('.') {
Some((label, number)) => (label, Some(number)),
None => (pre_release, None),
};
matches!(label, "alpha" | "beta" | "rc") && number.is_none_or(is_version_number)
}
#[test]
fn crate_version_has_the_supported_form() {
for version in [
"0.1.0",
"1.0.0",
"1.1.0-alpha",
"1.1.0-beta.2",
"1.1.0-rc.1",
] {
assert!(has_supported_version_form(version), "{version} must fit");
}
for version in [
"1.0.0-rc1",
"1.0.0-preview",
"1.0.0-RC.1",
"1.0.0+build.1",
"v1.0.0",
"1.0",
"01.0.0",
"1.0.0-rc.01",
] {
assert!(
!has_supported_version_form(version),
"{version} must not fit"
);
}
assert!(
has_supported_version_form(env!("CARGO_PKG_VERSION")),
"the crate version {} has an unsupported form",
env!("CARGO_PKG_VERSION")
);
}
}