#![cfg_attr(test, allow(clippy::unwrap_used))]
#![cfg_attr(test, allow(clippy::expect_used))]
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
mod readme_doctests {}
pub mod builders;
pub(crate) mod callback;
pub(crate) mod checkpoint_coalescer;
pub(crate) mod child;
pub(crate) mod client;
pub(crate) mod combinator;
mod context;
pub(crate) mod coordinator;
pub(crate) mod driver;
mod engine;
mod error;
mod future;
pub(crate) mod invoke;
pub(crate) mod map_parallel; pub mod observability;
mod operation;
mod operation_sub_type;
mod options;
pub mod plugin;
pub mod serdes;
pub(crate) mod start;
pub(crate) mod step;
mod task;
#[cfg(feature = "test-util")]
pub mod test_util;
pub(crate) mod tracing_layer;
pub(crate) mod wait;
pub(crate) mod wait_for_condition;
pub(crate) mod with_retry;
pub use self::tracing_layer::ReplayFilterLayer;
pub use self::client::{
CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
GetStateOutput, LambdaExecutionClient,
};
pub use self::context::{DurableContext, StepContext};
pub use self::error::{
CallbackError, CallbackErrorKind, ChildContextError, ChildContextErrorKind, CombinatorError,
CombinatorErrorKind, InvokeError, InvokeErrorKind, JoinFailed, MaxChecksExceeded,
NonDeterministicExecutionError, NonDeterministicExecutionErrorKind, OperationError,
OperationErrorKind, OperationMismatch, ReplayedFailure, StepError, StepErrorKind, TypedError,
UnexpectedStatus, UnrecognizedStatus, WaitError, WaitErrorKind, WaitForConditionError,
WaitForConditionErrorKind, WireError,
};
pub use self::future::{Branch, BranchHandle, BranchKey, BranchSet, BranchValue, DurableFuture};
pub use self::operation::{InvocationStatus, OperationKind, OperationStatus};
pub use self::operation_sub_type::{InvalidOperationSubType, OperationSubType};
pub use self::options::{Options, OptionsBuilder, OptionsValidationError};
pub use self::serdes::{JsonSerdes, PassthroughSerdes, PayloadSerializer, Serdes};
pub use self::step::StepSemantics;
pub use aws_config::SdkConfig;
pub use aws_sdk_lambda::Client as LambdaClient;
pub use lambda_runtime::{self, Context as LambdaContext};
pub mod lambda_types {
pub use aws_sdk_lambda::primitives::DateTime;
pub use aws_sdk_lambda::types::{
CallbackDetails, CallbackOptions, ChainedInvokeDetails, ChainedInvokeOptions,
ContextDetails, ContextOptions, ErrorObject, ExecutionDetails, Operation, OperationAction,
OperationStatus, OperationType, OperationUpdate, StepDetails, StepOptions, WaitDetails,
WaitOptions,
};
#[cfg(feature = "test-util")]
pub use aws_sdk_lambda::types::{
CallbackFailedDetails, CallbackStartedDetails, CallbackSucceededDetails,
CallbackTimedOutDetails, ChainedInvokeFailedDetails, ChainedInvokeStartedDetails,
ChainedInvokeStoppedDetails, ChainedInvokeSucceededDetails, ChainedInvokeTimedOutDetails,
ContextFailedDetails, ContextStartedDetails, ContextSucceededDetails, Event, EventError,
EventInput, EventResult, EventType, ExecutionFailedDetails, ExecutionStartedDetails,
ExecutionStoppedDetails, ExecutionSucceededDetails, ExecutionTimedOutDetails,
InvocationCompletedDetails, RetryDetails, StepFailedDetails, StepStartedDetails,
StepSucceededDetails, WaitCancelledDetails, WaitStartedDetails, WaitSucceededDetails,
};
}
use serde::{Deserialize, Serialize};
use std::future::Future;
use std::time::Duration;
pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
pub fn init_default_subscriber() -> Result<(), BoxError> {
init_default_subscriber_with_writer(std::io::stdout)
}
pub fn init_default_subscriber_with_writer<W>(writer: W) -> Result<(), BoxError>
where
W: for<'w> lambda_runtime::tracing::subscriber::fmt::MakeWriter<'w> + Send + Sync + 'static,
{
use lambda_runtime::tracing::subscriber::util::SubscriberInitExt as _;
let settings = tracing_layer::LogSettings::from_env();
tracing_layer::default_subscriber(
&settings.level,
&settings.directives,
&settings.format,
writer,
)
.try_init()?;
Ok(())
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RetryDecision {
#[non_exhaustive]
Retry {
delay: Duration,
},
Stop,
}
impl RetryDecision {
#[must_use]
pub fn retry(delay: Duration) -> Self {
Self::Retry { delay }
}
}
pub(crate) type RetryStrategy = Box<dyn Fn(&StepError, u32) -> RetryDecision + Send + Sync>;
pub async fn run<F, E, Fut, O>(handler: F) -> Result<(), lambda_runtime::Error>
where
F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
E: for<'de> Deserialize<'de> + Send + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send,
O: Serialize + Send + 'static,
{
run_with_options(handler, Options::default()).await
}
pub async fn run_with_options<F, E, Fut, O>(
handler: F,
options: Options,
) -> Result<(), lambda_runtime::Error>
where
F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
E: for<'de> Deserialize<'de> + Send + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send,
O: Serialize + Send + 'static,
{
lambda_runtime::run(lambda_runtime::service_fn(wrap(handler, options))).await
}
#[derive(Debug)]
struct InvocationEnvelope {
execution_arn: String,
checkpoint_token: String,
updated_operation_ids: Vec<String>,
}
fn has_envelope_shape(payload: &serde_json::Value) -> bool {
payload.get("DurableExecutionArn").is_some()
|| payload.get("CheckpointToken").is_some()
|| payload.get("InitialExecutionState").is_some()
}
fn parse_envelope(
payload: &serde_json::Value,
) -> Result<Option<InvocationEnvelope>, lambda_runtime::Error> {
if !has_envelope_shape(payload) {
return Ok(None);
}
let execution_arn = match payload.get("DurableExecutionArn") {
Some(v) => v
.as_str()
.ok_or_else(|| {
lambda_runtime::Error::from(
"malformed invocation envelope: \"DurableExecutionArn\" is present but is not \
a string"
.to_owned(),
)
})?
.to_owned(),
None => {
return Err(lambda_runtime::Error::from(
"malformed invocation envelope: required field \"DurableExecutionArn\" is missing"
.to_owned(),
));
}
};
let checkpoint_token = match payload.get("CheckpointToken") {
Some(v) => v
.as_str()
.ok_or_else(|| {
lambda_runtime::Error::from(
"malformed invocation envelope: \"CheckpointToken\" is present but is not a \
string"
.to_owned(),
)
})?
.to_owned(),
None => {
return Err(lambda_runtime::Error::from(
"malformed invocation envelope: required field \"CheckpointToken\" is missing"
.to_owned(),
));
}
};
let updated_operation_ids = payload
.get("UpdatedOperationIds")
.and_then(serde_json::Value::as_array)
.map(|ids| {
ids.iter()
.filter_map(serde_json::Value::as_str)
.map(str::to_owned)
.collect()
})
.unwrap_or_default();
Ok(Some(InvocationEnvelope {
execution_arn,
checkpoint_token,
updated_operation_ids,
}))
}
fn inline_input_payload(
payload: &serde_json::Value,
) -> Result<client::PageInput, lambda_runtime::Error> {
let execution_op = payload
.get("InitialExecutionState")
.and_then(|s| s.get("Operations"))
.and_then(serde_json::Value::as_array)
.and_then(|ops| {
ops.iter().find(|op| {
op.get("Type")
.and_then(serde_json::Value::as_str)
.is_some_and(|t| t.eq_ignore_ascii_case("EXECUTION"))
})
});
let Some(op) = execution_op else {
return Ok(client::PageInput::NoExecutionOperation);
};
match op
.get("ExecutionDetails")
.and_then(|d| d.get("InputPayload"))
{
None | Some(serde_json::Value::Null) => Ok(client::PageInput::Execution(None)),
Some(serde_json::Value::String(value)) => {
Ok(client::PageInput::Execution(Some(value.clone())))
}
Some(_) => Err(lambda_runtime::Error::from(
"malformed invocation envelope: InputPayload is not a string".to_owned(),
)),
}
}
fn extract_customer_input<E>(input_payload: Option<&str>) -> Result<E, lambda_runtime::Error>
where
E: for<'de> Deserialize<'de>,
{
serde_json::from_str(input_payload.unwrap_or("{}"))
.map_err(|e| lambda_runtime::Error::from(format!("deserialize customer input: {e}")))
}
fn wire_error_from_box_error(err: BoxError) -> WireError {
match err.downcast::<OperationError>() {
Ok(op_err) => wire_error_from_operation_error(&op_err),
Err(other) => error::wire_error_for(&*other, "Error"),
}
}
fn serialize_handler_result<T: Serialize>(result: &T) -> Result<String, WireError> {
serde_json::to_string(result).map_err(|e| error::wire_error_for(&e, "Error"))
}
fn wire_error_from_operation_error(err: &OperationError) -> WireError {
if let OperationErrorKind::Callback(cb_err) = err.kind()
&& matches!(cb_err.kind(), CallbackErrorKind::ExternalFailure)
&& let Some(wire) = err.wire()
{
return wire.clone();
}
error::wire_error_for(err, err.kind().wire_type_name())
}
fn parse_stack_trace(error: &serde_json::Value) -> Option<Vec<String>> {
let frames: Vec<String> = error
.get("StackTrace")?
.as_array()?
.iter()
.filter_map(serde_json::Value::as_str)
.map(String::from)
.collect();
if frames.is_empty() {
None
} else {
Some(frames)
}
}
fn parse_inline_operations(payload: &serde_json::Value) -> (engine::CheckpointLog, Option<String>) {
let initial_state = payload.get("InitialExecutionState");
let next_marker = initial_state
.and_then(|s| s.get("NextMarker"))
.and_then(serde_json::Value::as_str)
.filter(|m| !m.is_empty())
.map(String::from);
let records: Vec<(String, engine::CheckpointRecord)> = initial_state
.and_then(|s| s.get("Operations"))
.and_then(serde_json::Value::as_array)
.map(|ops| ops.iter().filter_map(parse_single_operation).collect())
.unwrap_or_default();
(engine::CheckpointLog::from_records(records), next_marker)
}
pub(crate) fn parse_envelope_timestamp(value: &serde_json::Value) -> Option<std::time::SystemTime> {
use aws_sdk_lambda::primitives::{DateTime, DateTimeFormat};
let parsed = match value {
serde_json::Value::String(s) => DateTime::from_str(s, DateTimeFormat::DateTime).ok()?,
serde_json::Value::Number(n) => DateTime::from_millis(n.as_i64()?),
_ => return None,
};
to_system_time(parsed)
}
pub(crate) fn to_system_time<T: TryInto<std::time::SystemTime>>(
timestamp: T,
) -> Option<std::time::SystemTime> {
timestamp.try_into().ok()
}
fn parse_inline_status(status_str: &str) -> engine::CheckpointStatus {
match status_str.to_ascii_uppercase().as_str() {
"STARTED" => engine::CheckpointStatus::Started,
"SUCCEEDED" => engine::CheckpointStatus::Succeeded,
"FAILED" => engine::CheckpointStatus::Failed,
"PENDING" => engine::CheckpointStatus::Pending,
"READY" => engine::CheckpointStatus::Ready,
"CANCELLED" => engine::CheckpointStatus::Cancelled,
"TIMED_OUT" => engine::CheckpointStatus::TimedOut,
"STOPPED" => engine::CheckpointStatus::Stopped,
_ => engine::CheckpointStatus::Unknown(status_str.to_owned()),
}
}
fn execution_info_from_json(op: &serde_json::Value) -> Option<plugin::OperationInfo> {
let id = op.get("Id").and_then(serde_json::Value::as_str)?;
let op_type = op.get("Type").and_then(serde_json::Value::as_str)?;
let status = parse_inline_status(
op.get("Status")
.and_then(serde_json::Value::as_str)
.unwrap_or("STARTED"),
);
plugin::OperationInfo::execution(
id,
op_type,
&status,
op.get("ParentId")
.and_then(serde_json::Value::as_str)
.map(String::from),
op.get("StartTimestamp").and_then(parse_envelope_timestamp),
op.get("EndTimestamp").and_then(parse_envelope_timestamp),
)
}
fn inline_bootstrap_state(
payload: &serde_json::Value,
) -> Result<(client::BootstrapState, Option<String>), lambda_runtime::Error> {
let input_payload = inline_input_payload(payload)?.into_payload();
let (log, next_marker) = parse_inline_operations(payload);
let state = client::BootstrapState {
log,
input_payload,
execution: inline_execution_info(payload),
};
Ok((state, next_marker))
}
fn inline_execution_info(payload: &serde_json::Value) -> Option<plugin::OperationInfo> {
payload
.get("InitialExecutionState")
.and_then(|s| s.get("Operations"))
.and_then(serde_json::Value::as_array)
.and_then(|ops| {
ops.iter().find(|op| {
op.get("Type")
.and_then(serde_json::Value::as_str)
.is_some_and(|t| t.eq_ignore_ascii_case("EXECUTION"))
})
})
.and_then(execution_info_from_json)
}
#[expect(clippy::too_many_lines)] fn parse_single_operation(op: &serde_json::Value) -> Option<(String, engine::CheckpointRecord)> {
let id = op.get("Id").and_then(serde_json::Value::as_str)?;
let op_type = op.get("Type").and_then(serde_json::Value::as_str)?;
if op_type.eq_ignore_ascii_case("Execution") {
return None;
}
let status_str = op
.get("Status")
.and_then(serde_json::Value::as_str)
.unwrap_or("STARTED");
let status = parse_inline_status(status_str);
let step_details = op.get("StepDetails");
let result = step_details
.and_then(|d| d.get("Result"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let error = step_details.and_then(|d| d.get("Error"));
let error_type = error
.and_then(|e| e.get("ErrorType"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let error_message = error
.and_then(|e| e.get("ErrorMessage"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let error_data = error
.and_then(|e| e.get("ErrorData"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let stack_trace = error.and_then(parse_stack_trace);
#[expect(clippy::cast_possible_truncation)] #[expect(clippy::cast_sign_loss)] let attempt = step_details
.and_then(|d| d.get("Attempt"))
.and_then(serde_json::Value::as_i64)
.map_or(0, |a| a.clamp(0, i64::from(u32::MAX)) as u32);
let due_at = op
.get("WaitDetails")
.and_then(|d| d.get("ScheduledEndTimestamp"))
.or_else(|| step_details.and_then(|d| d.get("NextAttemptTimestamp")))
.and_then(parse_envelope_timestamp);
let invoke_details = op.get("ChainedInvokeDetails");
let invoke_result = invoke_details
.and_then(|d| d.get("Result"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let invoke_error = invoke_details.and_then(|d| d.get("Error"));
let invoke_error_type = invoke_error
.and_then(|e| e.get("ErrorType"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let invoke_error_message = invoke_error
.and_then(|e| e.get("ErrorMessage"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let invoke_error_data = invoke_error
.and_then(|e| e.get("ErrorData"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let invoke_stack_trace = invoke_error.and_then(parse_stack_trace);
let context_details = op.get("ContextDetails");
let replay_children = context_details
.and_then(|d| d.get("ReplayChildren"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let callback_details = op.get("CallbackDetails");
let callback_id = callback_details
.and_then(|d| d.get("CallbackId"))
.and_then(serde_json::Value::as_str)
.map(String::from);
let result = result.or_else(|| {
context_details
.and_then(|d| d.get("Result"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let context_error = context_details.and_then(|d| d.get("Error"));
let error_type = error_type.or_else(|| {
context_error
.and_then(|e| e.get("ErrorType"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let error_message = error_message.or_else(|| {
context_error
.and_then(|e| e.get("ErrorMessage"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let error_data = error_data.or_else(|| {
context_error
.and_then(|e| e.get("ErrorData"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let stack_trace = stack_trace.or_else(|| context_error.and_then(parse_stack_trace));
let result = result.or_else(|| {
callback_details
.and_then(|d| d.get("Result"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let callback_error = callback_details.and_then(|d| d.get("Error"));
let error_type = error_type.or_else(|| {
callback_error
.and_then(|e| e.get("ErrorType"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let error_message = error_message.or_else(|| {
callback_error
.and_then(|e| e.get("ErrorMessage"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let error_data = error_data.or_else(|| {
callback_error
.and_then(|e| e.get("ErrorData"))
.and_then(serde_json::Value::as_str)
.map(String::from)
});
let stack_trace = stack_trace.or_else(|| callback_error.and_then(parse_stack_trace));
Some((
id.to_owned(),
engine::CheckpointRecord {
id: id.to_owned(),
status,
result,
error_type,
error_message,
error_data,
stack_trace,
attempt,
invoke_result,
invoke_error_type,
invoke_error_message,
invoke_error_data,
invoke_stack_trace,
replay_children,
callback_id,
op_type: Some(op_type.to_owned()),
sub_type: op
.get("SubType")
.and_then(serde_json::Value::as_str)
.map(String::from),
op_name: op
.get("Name")
.and_then(serde_json::Value::as_str)
.map(String::from),
due_at,
parent_id: op
.get("ParentId")
.and_then(serde_json::Value::as_str)
.map(String::from),
start_timestamp: op.get("StartTimestamp").and_then(parse_envelope_timestamp),
end_timestamp: op.get("EndTimestamp").and_then(parse_envelope_timestamp),
},
))
}
#[derive(Debug, serde::Deserialize)]
pub struct InvocationPayload(pub(crate) serde_json::Value);
#[derive(Debug, serde::Serialize)]
pub struct InvocationResponse(pub(crate) serde_json::Value);
pub(crate) type BoxedInvocationFuture = std::pin::Pin<
Box<dyn Future<Output = Result<InvocationResponse, lambda_runtime::Error>> + Send>,
>;
pub fn wrap<F, E, Fut, O>(
handler: F,
options: Options,
) -> impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture + Send + Sync
where
F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
E: for<'de> Deserialize<'de> + Send + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send,
O: Serialize + Send + 'static,
{
use std::sync::Arc as StdArc;
let Options {
sdk_config,
lambda_client,
execution_client,
checkpoint_delay,
plugins,
plugin_child_operations_depth,
} = options;
let checkpoint_buffer_window = checkpoint_delay.unwrap_or(Duration::ZERO);
let preset_client: Option<StdArc<dyn ExecutionClient>> = execution_client.or_else(|| {
base_lambda_client_from_options(sdk_config, lambda_client)
.map(|c| StdArc::new(LambdaExecutionClient::new(c)) as StdArc<dyn ExecutionClient>)
});
wrap_with_provider(
handler,
ClientProvider::new(preset_client),
checkpoint_buffer_window,
plugin_child_operations_depth,
plugins.into(),
)
}
#[expect(clippy::too_many_lines)] fn wrap_with_provider<F, E, Fut, O>(
handler: F,
provider: ClientProvider,
checkpoint_buffer_window: Duration,
plugin_child_operations_depth: u32,
plugin_factories: std::sync::Arc<plugin::Factories>,
) -> impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture + Send + Sync
where
F: Fn(E, DurableContext) -> Fut + Send + Sync + 'static,
E: for<'de> Deserialize<'de> + Send + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send,
O: Serialize + Send + 'static,
{
use std::sync::Arc as StdArc;
let handler = StdArc::new(handler);
let provider = StdArc::new(provider);
move |event: lambda_runtime::LambdaEvent<InvocationPayload>| -> std::pin::Pin<
Box<dyn Future<Output = Result<InvocationResponse, lambda_runtime::Error>> + Send>,
> {
let handler = StdArc::clone(&handler);
let provider = StdArc::clone(&provider);
let plugin_factories = StdArc::clone(&plugin_factories);
Box::pin(async move {
let (InvocationPayload(raw_payload), lambda_ctx) = event.into_parts();
let envelope = parse_envelope(&raw_payload)?.ok_or_else(|| {
lambda_runtime::Error::from(
"invocation payload is not a durable execution envelope \
(missing DurableExecutionArn, CheckpointToken, and \
InitialExecutionState)"
.to_owned(),
)
})?;
let execution_arn = envelope.execution_arn;
let checkpoint_token = envelope.checkpoint_token;
let updated_operation_ids = envelope.updated_operation_ids;
let (first_page, initial_marker) = inline_bootstrap_state(&raw_payload)?;
let exec_client = provider.get().await;
let (checkpoint_log, input_payload, execution) = match client::resolve_bootstrap_log(
exec_client.as_ref(),
&execution_arn,
&checkpoint_token,
first_page,
initial_marker.as_deref(),
)
.await
{
Ok(state) => (StdArc::new(state.log), state.input_payload, state.execution),
Err(e) if e.is_retryable() => {
return Err(lambda_runtime::Error::from(format!(
"failed to paginate initial state: {e}"
)));
}
Err(e) => {
return Ok(failed_envelope(&WireError::new(
Some(client::bootstrap_error_type(&e)),
Some(client::client_error_record_message(&e)),
)));
}
};
let customer_input: E = extract_customer_input(input_payload.as_deref())?;
let plugins = create_invocation_plugins(
&plugin_factories,
&lambda_ctx,
&execution_arn,
input_payload.as_deref(),
&checkpoint_log,
execution.as_ref(),
updated_operation_ids,
);
let _shut_down = plugins.as_ref().map(plugin::ShutDownOnDrop::new);
if let Some(plugins) = &plugins {
plugins.invocation_start().await;
}
let end_log = StdArc::clone(&checkpoint_log);
let ctx = DurableContext::new_root_with_client_and_defaults(
execution_arn,
lambda_ctx,
checkpoint_log,
exec_client,
checkpoint_token,
checkpoint_buffer_window,
plugin_child_operations_depth,
plugins.clone(),
);
let flush_ctx = ctx.clone();
let coordinator = StdArc::clone(ctx.coordinator());
if let Some(plugins) = &plugins {
plugins.attach(&coordinator);
}
let remaining = ctx.remaining_time();
let replay_span = ctx.replay_span();
{
let _execution_scope = replay_span.enter();
tracing_layer::invocation_begin_event(
ctx.is_replaying(),
ctx.execution_arn(),
&ctx.lambda_context().request_id,
);
}
let task_ownership = StdArc::clone(ctx.task_ownership());
let root = context::run_body(context::ActiveBody::Context(String::new()), async move {
match (handler)(customer_input, ctx).await {
Ok(result) => serialize_handler_result(&result),
Err(e) => Err(wire_error_from_box_error(e)),
}
});
let frames = plugin::Frames::invocation(plugins.clone());
let root_cx = task::TaskContext::new(frames.clone(), replay_span);
let finished = plugin::Frames::instrument(
&frames,
CatchFinished::new(finish_invocation(
root,
root_cx,
task_ownership,
coordinator,
&flush_ctx,
remaining,
plugins.as_deref(),
&end_log,
)),
)
.await;
drop(frames);
respond(finished, flush_ctx, plugins)
})
}
}
#[cfg(test)]
mod test_hooks {
tokio::task_local! {
pub(crate) static PENDING_AFTER_DRIVE:
std::sync::Arc<std::sync::atomic::AtomicBool>;
}
pub(crate) async fn pending_once_after_drive() {
let Ok(flag) = PENDING_AFTER_DRIVE.try_with(std::sync::Arc::clone) else {
return;
};
let mut pending = true;
std::future::poll_fn(|cx| {
if std::mem::take(&mut pending) {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
cx.waker().wake_by_ref();
std::task::Poll::Pending
} else {
std::task::Poll::Ready(())
}
})
.await;
}
pub(crate) struct HeldGuard {
pub(crate) reached: std::sync::Arc<std::sync::atomic::AtomicBool>,
pub(crate) make: Box<dyn Fn() -> Box<dyn Send> + Send + Sync>,
}
tokio::task_local! {
pub(crate) static PENDING_HOLDING_GUARD: std::sync::Arc<HeldGuard>;
}
pub(crate) async fn pending_holding_guard() {
let Ok(held) = PENDING_HOLDING_GUARD.try_with(std::sync::Arc::clone) else {
return;
};
let guard = (held.make)();
let mut pending = true;
std::future::poll_fn(|cx| {
if std::mem::take(&mut pending) {
held.reached
.store(true, std::sync::atomic::Ordering::SeqCst);
cx.waker().wake_by_ref();
std::task::Poll::Pending
} else {
std::task::Poll::Ready(())
}
})
.await;
drop(guard);
}
}
enum Finished {
Respond(
Result<InvocationResponse, lambda_runtime::Error>,
Option<Box<dyn std::any::Any + Send + 'static>>,
),
Panicked(Box<dyn std::any::Any + Send + 'static>),
}
#[expect(
clippy::too_many_arguments,
reason = "the root task's future and its context are passed apart, as `spawn_root` takes them"
)]
async fn finish_invocation<R>(
root: R,
root_cx: task::TaskContext,
task_ownership: std::sync::Arc<driver::TaskOwnership>,
coordinator: std::sync::Arc<coordinator::InvocationCoordinator>,
flush_ctx: &DurableContext,
remaining: Option<Duration>,
plugins: Option<&plugin::InvocationPlugins>,
end_log: &engine::CheckpointLog,
) -> Finished
where
R: Future<Output = Result<String, WireError>> + Send + 'static,
{
let (outcome, fatal_won) = match driver::drive_invocation(
root,
root_cx,
task_ownership,
coordinator,
flush_ctx.clone(),
remaining,
)
.await
{
Ok(driven) => driven,
Err(payload) => {
let end = InvocationEnd::panicked(&*payload);
end_invocation(plugins, end_log, flush_ctx, end, None, false)
.await
.into_iter()
.for_each(task::discard_panic);
return Finished::Panicked(payload);
}
};
#[cfg(test)]
test_hooks::pending_once_after_drive().await;
#[cfg(test)]
test_hooks::pending_holding_guard().await;
if !matches!(outcome, driver::InvocationOutcome::Fault { .. }) {
flush_ctx.flush_pending_checkpoints().await;
}
let outcome = match flush_ctx.coordinator().write_failure() {
Some(failure) if !fatal_won => failure.into_outcome(),
_ => outcome,
};
let (response, end) = match outcome {
driver::InvocationOutcome::Complete(serialized)
if serialized.len() > LAMBDA_RESPONSE_SIZE_LIMIT =>
{
checkpoint_oversized_result(flush_ctx, serialized).await
}
outcome => outcome_envelope(outcome, flush_ctx),
};
let (response, end) = driver::withdrawal_answer(flush_ctx, response, end);
let late = end_invocation(plugins, end_log, flush_ctx, end, Some(&response), fatal_won).await;
Finished::Respond(response, late)
}
fn respond(
finished: Finished,
ctx: DurableContext,
plugins: Option<std::sync::Arc<plugin::InvocationPlugins>>,
) -> Result<InvocationResponse, lambda_runtime::Error> {
let payload = match finished {
Finished::Respond(response, None) => return response,
Finished::Respond(response, Some(payload)) => {
drop_contained(response);
payload
}
Finished::Panicked(payload) => payload,
};
drop_contained(ctx);
drop_contained(plugins);
std::panic::resume_unwind(payload)
}
fn drop_contained<T>(value: T) {
if let Err(payload) =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || drop(value)))
{
task::discard_panic(payload);
}
}
struct CatchFinished<F> {
inner: Option<std::pin::Pin<Box<F>>>,
}
impl<F> CatchFinished<F> {
fn new(inner: F) -> Self {
Self {
inner: Some(Box::pin(inner)),
}
}
}
impl<F: Future<Output = Finished>> Future for CatchFinished<F> {
type Output = Finished;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Finished> {
let this = self.get_mut();
let Some(inner) = this.inner.as_mut() else {
return std::task::Poll::Pending;
};
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| inner.as_mut().poll(cx))) {
Ok(std::task::Poll::Pending) => std::task::Poll::Pending,
Ok(std::task::Poll::Ready(finished)) => {
this.inner = None;
std::task::Poll::Ready(finished)
}
Err(payload) => {
if let Some(inner) = this.inner.take() {
drop_contained(inner);
}
std::task::Poll::Ready(Finished::Panicked(payload))
}
}
}
}
pub(crate) struct InvocationEnd {
pub(crate) status: InvocationStatus,
pub(crate) result: Option<String>,
pub(crate) error: Option<WireError>,
}
impl InvocationEnd {
pub(crate) fn panicked(payload: &(dyn std::any::Any + Send)) -> Self {
Self {
status: InvocationStatus::Retrying,
result: None,
error: Some(WireError::new(
None::<String>,
Some(panic_payload_text(payload)),
)),
}
}
fn retrying(error_type: impl Into<String>, message: String) -> Self {
Self {
status: InvocationStatus::Retrying,
result: None,
error: Some(WireError::new(Some(error_type.into()), Some(message))),
}
}
}
pub(crate) fn panic_payload_text(payload: &(dyn std::any::Any + Send)) -> String {
if let Some(text) = payload.downcast_ref::<&'static str>() {
(*text).to_owned()
} else if let Some(text) = payload.downcast_ref::<String>() {
text.clone()
} else {
"the invocation panicked with a payload that is not a string".to_owned()
}
}
async fn end_invocation(
plugins: Option<&plugin::InvocationPlugins>,
log: &engine::CheckpointLog,
ctx: &DurableContext,
end: InvocationEnd,
response: Option<&Result<InvocationResponse, lambda_runtime::Error>>,
fatal_won: bool,
) -> Option<Box<dyn std::any::Any + Send + 'static>> {
let plugins = plugins?;
ctx.report_operation_changes().await;
plugins
.invocation_end(
log,
end,
ctx.remaining_time(),
late_panics(response, fatal_won),
)
.await
}
fn late_panics(
response: Option<&Result<InvocationResponse, lambda_runtime::Error>>,
fatal_won: bool,
) -> plugin::LatePanics {
match response {
Some(Ok(_)) if !fatal_won => plugin::LatePanics::Fail,
_ => plugin::LatePanics::Discard,
}
}
fn create_invocation_plugins(
factories: &plugin::Factories,
lambda_ctx: &lambda_runtime::Context,
execution_arn: &str,
input_payload: Option<&str>,
checkpoint_log: &engine::CheckpointLog,
execution: Option<&plugin::OperationInfo>,
updated_operation_ids: Vec<String>,
) -> Option<std::sync::Arc<plugin::InvocationPlugins>> {
plugin::create_plugins(factories, execution.cloned(), || {
plugin::InvocationInfo::new(
lambda_ctx.request_id.clone(),
execution_arn.to_owned(),
input_payload.unwrap_or("{}").to_owned(),
plugin::operations_map(checkpoint_log, execution),
updated_operation_ids,
execution.and_then(plugin::OperationInfo::start_timestamp),
!checkpoint_log.has_records(),
lambda_ctx.xray_trace_id.clone(),
)
})
}
pub(crate) fn outcome_envelope(
outcome: driver::InvocationOutcome,
ctx: &DurableContext,
) -> (
Result<InvocationResponse, lambda_runtime::Error>,
InvocationEnd,
) {
match outcome {
driver::InvocationOutcome::Complete(serialized) => (
Ok(InvocationResponse(serde_json::json!({
"Status": "SUCCEEDED",
"Result": serialized
}))),
InvocationEnd {
status: InvocationStatus::Succeeded,
result: Some(serialized),
error: None,
},
),
driver::InvocationOutcome::Pending => {
let execution_span = ctx.replay_span();
let _execution_scope = execution_span.enter();
tracing_layer::execution_suspended_event(
ctx.execution_arn(),
&ctx.lambda_context().request_id,
);
(
Ok(InvocationResponse(serde_json::json!({
"Status": "PENDING"
}))),
InvocationEnd {
status: InvocationStatus::Pending,
result: None,
error: None,
},
)
}
driver::InvocationOutcome::Fault {
error_type,
message,
} => {
(
Err(lambda_runtime::Error::from(message.clone())),
InvocationEnd::retrying(error_type, message),
)
}
driver::InvocationOutcome::Failed { error } => (
Ok(failed_envelope(&error)),
InvocationEnd {
status: InvocationStatus::Failed,
result: None,
error: Some(error),
},
),
}
}
const LAMBDA_RESPONSE_SIZE_LIMIT: usize = 6 * 1024 * 1024 - 50;
async fn checkpoint_oversized_result(
ctx: &DurableContext,
serialized: String,
) -> (
Result<InvocationResponse, lambda_runtime::Error>,
InvocationEnd,
) {
#[expect(clippy::expect_used)]
let epoch_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("the clock is after the Unix epoch")
.as_millis();
#[expect(clippy::expect_used)]
let update = aws_sdk_lambda::types::OperationUpdate::builder()
.id(engine::compute_wire_id_public(&format!(
"execution-result-{epoch_ms}"
)))
.r#type(aws_sdk_lambda::types::OperationType::Execution)
.action(aws_sdk_lambda::types::OperationAction::Succeed)
.payload(serialized.clone())
.build()
.expect("the update's required fields are set");
match ctx.checkpoint_updates_after_drive(vec![update]).await {
Ok(_) => (
Ok(InvocationResponse(serde_json::json!({
"Status": "SUCCEEDED",
"Result": ""
}))),
InvocationEnd {
status: InvocationStatus::Succeeded,
result: Some(serialized),
error: None,
},
),
Err(context::WriteError::Failed(e)) if e.is_retryable() => {
let message = error::checkpoint_invocation_failure_message(&e);
(
Err(lambda_runtime::Error::from(message.clone())),
InvocationEnd::retrying(error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE, message),
)
}
Err(context::WriteError::Failed(e)) => {
let error = error::checkpoint_execution_failure_wire(&e);
(
Ok(failed_envelope(&error)),
InvocationEnd {
status: InvocationStatus::Failed,
result: None,
error: Some(error),
},
)
}
Err(context::WriteError::Stopped) => {
outcome_envelope(driver::InvocationOutcome::Pending, ctx)
}
}
}
fn failed_envelope(error: &WireError) -> InvocationResponse {
let mut error_map = serde_json::Map::new();
error_map.insert(
"ErrorType".to_owned(),
serde_json::Value::String(error.error_type().unwrap_or("Error").to_owned()),
);
error_map.insert(
"ErrorMessage".to_owned(),
serde_json::Value::String(error.error_message().unwrap_or_default().to_owned()),
);
if let Some(data) = error.error_data() {
error_map.insert(
"ErrorData".to_owned(),
serde_json::Value::String(data.to_owned()),
);
}
if !error.stack_trace().is_empty() {
error_map.insert(
"StackTrace".to_owned(),
serde_json::json!(error.stack_trace()),
);
}
InvocationResponse(serde_json::json!({
"Status": "FAILED",
"Error": serde_json::Value::Object(error_map)
}))
}
pub(crate) fn base_lambda_client_from_options(
sdk_config: Option<SdkConfig>,
lambda_client: Option<aws_sdk_lambda::Client>,
) -> Option<aws_sdk_lambda::Client> {
match (lambda_client, sdk_config) {
(Some(client), _) => Some(client),
(None, Some(config)) => Some(aws_sdk_lambda::Client::new(&config)),
(None, None) => None,
}
}
const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5);
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(50);
const DEFAULT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(55);
fn default_lambda_client(config: &SdkConfig) -> aws_sdk_lambda::Client {
use aws_sdk_lambda::config::timeout::TimeoutConfig;
let conf = aws_sdk_lambda::config::Builder::from(config)
.timeout_config(
TimeoutConfig::builder()
.connect_timeout(DEFAULT_CONNECT_TIMEOUT)
.read_timeout(DEFAULT_READ_TIMEOUT)
.operation_attempt_timeout(DEFAULT_ATTEMPT_TIMEOUT)
.build(),
)
.build();
aws_sdk_lambda::Client::from_conf(conf)
}
pub(crate) struct ClientProvider {
preset: Option<std::sync::Arc<dyn ExecutionClient>>,
default_cell: tokio::sync::OnceCell<std::sync::Arc<dyn ExecutionClient>>,
}
impl ClientProvider {
pub(crate) fn new(preset: Option<std::sync::Arc<dyn ExecutionClient>>) -> Self {
Self {
preset,
default_cell: tokio::sync::OnceCell::new(),
}
}
pub(crate) async fn get(&self) -> std::sync::Arc<dyn ExecutionClient> {
use std::sync::Arc as StdArc;
if let Some(preset) = &self.preset {
return StdArc::clone(preset);
}
let client = self
.default_cell
.get_or_init(|| async {
let aws_config =
aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
let lambda_client = default_lambda_client(&aws_config);
StdArc::new(LambdaExecutionClient::new(lambda_client))
as StdArc<dyn ExecutionClient>
})
.await;
StdArc::clone(client)
}
}
#[cfg(test)]
mod tests {
use std::future::IntoFuture;
use super::*;
#[test]
fn retry_decision_constructor_builds_retry() {
assert!(matches!(
RetryDecision::retry(Duration::from_secs(3)),
RetryDecision::Retry { delay } if delay == Duration::from_secs(3)
));
}
#[test]
fn due_at_from_inline_timestamps() {
let payload = serde_json::json!({
"InitialExecutionState": {
"Operations": [
{ "Id": "w1", "Type": "WAIT", "Status": "STARTED",
"WaitDetails": { "ScheduledEndTimestamp": "2026-01-01T00:00:00Z" } },
{ "Id": "s1", "Type": "STEP", "Status": "PENDING",
"StepDetails": { "NextAttemptTimestamp": "2026-01-01T00:00:05.5Z" } },
{ "Id": "w2", "Type": "WAIT", "Status": "STARTED",
"WaitDetails": { "ScheduledEndTimestamp": 1_767_225_600_000_i64 } },
{ "Id": "w3", "Type": "WAIT", "Status": "STARTED",
"WaitDetails": { "ScheduledEndTimestamp": "not a date" } },
{ "Id": "w4", "Type": "WAIT", "Status": "STARTED",
"WaitDetails": { "ScheduledEndTimestamp": 1.5 } },
{ "Id": "w5", "Type": "WAIT", "Status": "STARTED",
"WaitDetails": { "ScheduledEndTimestamp": true } },
]
}
});
let (log, _) = parse_inline_operations(&payload);
let due = |id: &str| log.get(id).expect("parsed").due_at;
let new_year = std::time::UNIX_EPOCH + Duration::from_hours(490_896);
assert_eq!(due("w1"), Some(new_year));
assert_eq!(due("s1"), Some(new_year + Duration::from_millis(5_500)));
assert_eq!(due("w2"), Some(new_year), "epoch milliseconds");
assert_eq!(due("w3"), None, "a string that does not parse");
assert_eq!(due("w4"), None, "a number that is not an integer");
assert_eq!(due("w5"), None, "another JSON type");
}
struct OutOfRange;
impl TryFrom<OutOfRange> for std::time::SystemTime {
type Error = &'static str;
fn try_from(_: OutOfRange) -> Result<Self, Self::Error> {
Err("outside the range of SystemTime")
}
}
#[test]
fn to_system_time_maps_an_out_of_range_timestamp_to_none() {
let millis = aws_sdk_lambda::primitives::DateTime::from_millis(1_500);
assert_eq!(
to_system_time(millis),
Some(std::time::UNIX_EPOCH + Duration::from_millis(1_500))
);
assert_eq!(to_system_time(OutOfRange), None);
}
#[derive(Debug)]
struct PlainUserError;
impl std::fmt::Display for PlainUserError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "plain user failure")
}
}
impl std::error::Error for PlainUserError {}
#[test]
fn handler_error_records_no_stack_trace() {
let wire = wire_error_from_box_error(Box::new(PlainUserError));
assert_eq!(wire.error_type(), Some("Error"));
assert_eq!(wire.error_message(), Some("plain user failure"));
assert_eq!(wire.stack_trace(), [] as [String; 0]);
}
struct Refuse;
impl Serialize for Refuse {
fn serialize<S: serde::Serializer>(&self, _serializer: S) -> Result<S::Ok, S::Error> {
Err(<S::Error as serde::ser::Error>::custom("refused"))
}
}
#[test]
fn handler_result_serialization_failure_records_error() {
let Err(wire) = serialize_handler_result(&Refuse) else {
unreachable!("a refusing value must not serialize");
};
assert_eq!(wire.error_type(), Some("Error"));
assert!(
wire.error_message().unwrap_or_default().contains("refused"),
"{:?}",
wire.error_message()
);
assert_eq!(wire.stack_trace(), [] as [String; 0]);
}
#[tokio::test]
async fn tokio_join_accepts_into_future() {
fn check_into_future<T: IntoFuture>(_t: T) {}
let ctx = DurableContext::test_context();
check_into_future(ctx.step(|_| async { Ok(1i32) }));
}
#[test]
fn wrap_returns_callable_service_function() {
fn assert_send_sync<T: Send + Sync>(_t: &T) {}
let service = wrap(
|_event: serde_json::Value, _ctx: DurableContext| async move {
Ok::<String, BoxError>("hello".to_owned())
},
Options::default(),
);
assert_send_sync(&service);
drop(service);
}
fn offline_options() -> Options {
let conf = aws_sdk_lambda::config::Config::builder()
.behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
.region(aws_sdk_lambda::config::Region::new("us-east-1"))
.build();
Options::builder()
.lambda_client(aws_sdk_lambda::Client::from_conf(conf))
.build()
.expect("offline options build")
}
async fn invoke_wrap_service(
payload: serde_json::Value,
) -> Result<serde_json::Value, lambda_runtime::Error> {
let service = wrap(
|event: serde_json::Value, _ctx: DurableContext| async move {
Ok::<serde_json::Value, BoxError>(event)
},
offline_options(),
);
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_runtime::Context::default(),
);
service(event).await.map(|response| response.0)
}
#[tokio::test]
async fn wrap_service_missing_arn_fails() {
let payload = serde_json::json!({
"CheckpointToken": "token-abc",
"InitialExecutionState": { "Operations": [] }
});
let err = invoke_wrap_service(payload).await.expect_err("must fail");
assert!(
err.to_string().contains("DurableExecutionArn"),
"error should name the missing field, got: {err}"
);
}
#[tokio::test]
async fn wrap_service_missing_token_fails() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:123456789012:function:test",
"InitialExecutionState": { "Operations": [] }
});
let err = invoke_wrap_service(payload).await.expect_err("must fail");
assert!(
err.to_string().contains("CheckpointToken"),
"error should name the missing field, got: {err}"
);
}
#[tokio::test]
async fn wrap_service_envelope_free_payload_fails() {
let payload = serde_json::json!({ "count": 42 });
let err = invoke_wrap_service(payload).await.expect_err("must fail");
assert!(
err.to_string().contains("not a durable execution envelope"),
"error should describe the missing envelope, got: {err}"
);
}
#[tokio::test]
async fn wrap_service_valid_envelope_succeeds() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:123456789012:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": {
"Operations": [{
"Id": "root",
"Type": "Execution",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{\"count\":42}" }
}]
}
});
let response = invoke_wrap_service(payload).await.expect("must succeed");
assert_eq!(
response.get("Status").and_then(serde_json::Value::as_str),
Some("SUCCEEDED"),
"unexpected response: {response}"
);
let result_json = response
.get("Result")
.and_then(serde_json::Value::as_str)
.expect("Result should be a serialized JSON string");
let echoed: serde_json::Value = serde_json::from_str(result_json).unwrap();
assert_eq!(echoed, serde_json::json!({ "count": 42 }));
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn handler_panic_leaves_sdk_with_original_payload() {
#[expect(clippy::panic)] fn boom() -> Result<(), BoxError> {
panic!("root boom")
}
let client: std::sync::Arc<dyn ExecutionClient> =
std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
let service = wrap_with_provider(
|_event: serde_json::Value, _ctx: DurableContext| async move { boom() },
ClientProvider::new(Some(client)),
Duration::ZERO,
0,
std::sync::Arc::from(Vec::new()),
);
let payload = serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": {
"Operations": [{
"Id": "root",
"Type": "Execution",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}]
}
});
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_runtime::Context::default(),
);
let join_err = tokio::spawn(service(event))
.await
.expect_err("the handler panic leaves the service, and no envelope is produced");
assert!(join_err.is_panic(), "a panic, not a cancellation");
let payload = join_err.into_panic();
assert_eq!(payload.downcast_ref::<&str>(), Some(&"root boom"));
}
fn paginated_envelope() -> serde_json::Value {
serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": {
"Operations": [{
"Id": "root",
"Type": "Execution",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}],
"NextMarker": "page-2"
}
})
}
async fn invoke_paginated(
sdk_client: aws_sdk_lambda::Client,
) -> (Result<serde_json::Value, lambda_runtime::Error>, bool) {
invoke_paginated_with(
Options::builder()
.lambda_client(sdk_client)
.build()
.expect("options build"),
)
.await
}
async fn invoke_paginated_with(
options: Options,
) -> (Result<serde_json::Value, lambda_runtime::Error>, bool) {
use std::sync::atomic::{AtomicBool, Ordering};
let ran = std::sync::Arc::new(AtomicBool::new(false));
let handler_ran = std::sync::Arc::clone(&ran);
let service = wrap(
move |_event: serde_json::Value, _ctx: DurableContext| {
handler_ran.store(true, Ordering::SeqCst);
async move { Ok::<(), BoxError>(()) }
},
options,
);
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(paginated_envelope()),
lambda_runtime::Context::default(),
);
let result = service(event).await.map(|response| response.0);
(result, ran.load(Ordering::SeqCst))
}
#[tokio::test]
async fn bootstrap_kms_error_returns_failed_envelope() {
use aws_smithy_mocks::{mock, mock_client};
let rule =
mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_http_response(|| {
client::mock_aws_error_response(400, "KMSAccessDeniedException", "kms denied")
});
let (result, ran) = invoke_paginated(mock_client!(aws_sdk_lambda, [&rule])).await;
let response = result.expect("a KMS error returns an envelope");
assert_eq!(
response.get("Status").and_then(serde_json::Value::as_str),
Some("FAILED"),
"unexpected response: {response}"
);
let error = response
.get("Error")
.expect("a FAILED envelope has an Error");
assert_eq!(
error.get("ErrorType").and_then(serde_json::Value::as_str),
Some("KMSAccessDeniedException")
);
assert_eq!(
error
.get("ErrorMessage")
.and_then(serde_json::Value::as_str),
Some("kms denied")
);
assert!(!ran, "the handler must not run");
}
#[tokio::test]
async fn bootstrap_kms_error_records_aws_error_code() {
use aws_smithy_mocks::{mock, mock_client};
let rule =
mock!(aws_sdk_lambda::Client::get_durable_execution_state).then_http_response(|| {
client::mock_aws_error_response(400, "KMSAccessDeniedException", "kms denied")
});
let client = mock_client!(aws_sdk_lambda, [&rule]);
let (result, ran) = invoke_paginated_with(
Options::builder()
.execution_client(std::sync::Arc::new(LambdaExecutionClient::new(client)))
.build()
.expect("options build"),
)
.await;
let response = result.expect("a KMS error returns an envelope");
assert_eq!(
response.get("Status").and_then(serde_json::Value::as_str),
Some("FAILED"),
"unexpected response: {response}"
);
let error = response
.get("Error")
.expect("a FAILED envelope has an Error");
assert_eq!(
error.get("ErrorType").and_then(serde_json::Value::as_str),
Some("KMSAccessDeniedException")
);
assert_eq!(
error
.get("ErrorMessage")
.and_then(serde_json::Value::as_str),
Some("kms denied")
);
assert!(!ran, "the handler must not run");
}
#[tokio::test]
async fn lambda_checkpoint_rejection_records_the_service_message() {
use aws_sdk_lambda::operation::checkpoint_durable_execution::CheckpointDurableExecutionOutput;
use aws_smithy_mocks::{RuleMode, mock, mock_client};
let rejected = mock!(aws_sdk_lambda::Client::checkpoint_durable_execution)
.then_http_response(|| {
client::mock_aws_error_response(
400,
"InvalidParameterValueException",
"bad payload",
)
});
let accepted =
mock!(aws_sdk_lambda::Client::checkpoint_durable_execution).then_output(|| {
CheckpointDurableExecutionOutput::builder()
.checkpoint_token("tok-2")
.build()
});
let client = mock_client!(aws_sdk_lambda, RuleMode::Sequential, [&rejected, &accepted]);
let service = wrap(
|_event: serde_json::Value, ctx: DurableContext| async move {
let value: u32 = ctx.step(|_| async { Ok(7_u32) }).name("one").await?;
Ok::<u32, BoxError>(value)
},
Options::builder()
.execution_client(std::sync::Arc::new(LambdaExecutionClient::new(client)))
.build()
.expect("options build"),
);
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": {
"Operations": [{
"Id": "root",
"Type": "Execution",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}]
}
})),
lambda_runtime::Context::default(),
);
let response = service(event)
.await
.map(|response| response.0)
.expect("a rejected checkpoint returns an envelope");
assert_eq!(
response.get("Status").and_then(serde_json::Value::as_str),
Some("FAILED"),
"unexpected response: {response}"
);
let error = response
.get("Error")
.expect("a FAILED envelope has an Error");
assert_eq!(
error.get("ErrorType").and_then(serde_json::Value::as_str),
Some("CheckpointUnrecoverableExecutionError")
);
assert_eq!(
error
.get("ErrorMessage")
.and_then(serde_json::Value::as_str),
Some("[Unrecoverable Execution] Checkpoint failed: bad payload")
);
}
#[tokio::test]
async fn bootstrap_retryable_error_fails_the_invocation() {
use aws_smithy_mocks::{mock, mock_client};
for status in [503, 403] {
let rule = mock!(aws_sdk_lambda::Client::get_durable_execution_state)
.sequence()
.http_status(status, None)
.repeatedly()
.build();
let (result, ran) = invoke_paginated(mock_client!(aws_sdk_lambda, [&rule])).await;
let err = result.expect_err("a retryable error fails the invocation");
assert!(
err.to_string().contains("failed to paginate initial state"),
"status {status}: {err}"
);
assert!(!ran, "the handler must not run");
}
}
fn step_op_with_status(status: &str) -> serde_json::Value {
serde_json::json!({
"Id": "op-1",
"Type": "Step",
"Status": status,
"StepDetails": {}
})
}
#[test]
fn parse_status_timed_out_wire_value_maps_to_timed_out() {
let (_, record) = parse_single_operation(&step_op_with_status("TIMED_OUT")).unwrap();
assert_eq!(record.status, engine::CheckpointStatus::TimedOut);
}
#[test]
fn parse_status_timedout_without_underscore_is_unrecognized() {
let (_, record) = parse_single_operation(&step_op_with_status("TIMEDOUT")).unwrap();
assert_eq!(
record.status,
engine::CheckpointStatus::Unknown("TIMEDOUT".to_owned())
);
}
#[test]
fn parse_unknown_status_carries_raw_value() {
let (_, record) = parse_single_operation(&step_op_with_status("Paused")).unwrap();
assert_eq!(
record.status,
engine::CheckpointStatus::Unknown("Paused".to_owned())
);
}
#[test]
fn parse_known_statuses_map_explicitly() {
let cases = [
("STARTED", engine::CheckpointStatus::Started),
("PENDING", engine::CheckpointStatus::Pending),
("READY", engine::CheckpointStatus::Ready),
("SUCCEEDED", engine::CheckpointStatus::Succeeded),
("FAILED", engine::CheckpointStatus::Failed),
("CANCELLED", engine::CheckpointStatus::Cancelled),
("TIMED_OUT", engine::CheckpointStatus::TimedOut),
("STOPPED", engine::CheckpointStatus::Stopped),
("Succeeded", engine::CheckpointStatus::Succeeded),
];
for (wire, expected) in cases {
let (_, record) = parse_single_operation(&step_op_with_status(wire)).unwrap();
assert_eq!(record.status, expected, "wire status {wire:?}");
}
}
#[test]
fn parse_callback_details_extracts_result() {
let op = serde_json::json!({
"Id": "abc123",
"Type": "Callback",
"Status": "SUCCEEDED",
"CallbackDetails": {
"CallbackId": "cb-42",
"Result": "\"hello from callback\""
}
});
let parsed = parse_single_operation(&op);
assert!(parsed.is_some());
let (id, record) = parsed.unwrap();
assert_eq!(id, "abc123");
assert_eq!(record.status, engine::CheckpointStatus::Succeeded);
assert_eq!(record.callback_id.as_deref(), Some("cb-42"));
assert_eq!(record.result.as_deref(), Some("\"hello from callback\""));
}
#[test]
fn parse_callback_details_extracts_error() {
let op = serde_json::json!({
"Id": "abc456",
"Type": "Callback",
"Status": "FAILED",
"CallbackDetails": {
"CallbackId": "cb-99",
"Error": {
"ErrorType": "NotApproved",
"ErrorMessage": "request was denied"
}
}
});
let parsed = parse_single_operation(&op);
assert!(parsed.is_some());
let (id, record) = parsed.unwrap();
assert_eq!(id, "abc456");
assert_eq!(record.status, engine::CheckpointStatus::Failed);
assert_eq!(record.callback_id.as_deref(), Some("cb-99"));
assert_eq!(record.error_type.as_deref(), Some("NotApproved"));
assert_eq!(record.error_message.as_deref(), Some("request was denied"));
}
#[test]
fn parse_callback_details_result_does_not_override_step_result() {
let op = serde_json::json!({
"Id": "abc789",
"Type": "Callback",
"Status": "SUCCEEDED",
"StepDetails": {
"Result": "\"from step\""
},
"CallbackDetails": {
"CallbackId": "cb-1",
"Result": "\"from callback\""
}
});
let parsed = parse_single_operation(&op);
assert!(parsed.is_some());
let (_, record) = parsed.unwrap();
assert_eq!(record.result.as_deref(), Some("\"from step\""));
}
#[test]
fn parse_inline_operations_handles_callback_succeeded() {
let payload = serde_json::json!({
"InitialExecutionState": {
"Operations": [
{
"Id": "exec-0",
"Type": "Execution",
"Status": "STARTED"
},
{
"Id": "wire-id-1",
"Type": "Callback",
"Status": "SUCCEEDED",
"CallbackDetails": {
"CallbackId": "cb-id-123",
"Result": "\"payload\""
}
}
]
}
});
let (log, marker) = parse_inline_operations(&payload);
assert!(marker.is_none());
let record = log.get("wire-id-1");
assert!(record.is_some());
let record = record.unwrap();
assert_eq!(record.callback_id.as_deref(), Some("cb-id-123"));
assert_eq!(record.result.as_deref(), Some("\"payload\""));
}
#[test]
fn parse_inline_operations_returns_next_marker() {
let payload = serde_json::json!({
"InitialExecutionState": {
"Operations": [
{
"Id": "exec-0",
"Type": "Execution",
"Status": "STARTED"
},
{
"Id": "wire-id-1",
"Type": "Step",
"Status": "SUCCEEDED",
"StepDetails": {
"Attempt": 1,
"Result": "\"hello\""
}
}
],
"NextMarker": "page-token-2"
}
});
let (log, marker) = parse_inline_operations(&payload);
let record = log.get("wire-id-1");
assert!(record.is_some());
assert_eq!(marker, Some("page-token-2".to_owned()));
}
#[test]
fn parse_inline_operations_ignores_empty_marker() {
let payload = serde_json::json!({
"InitialExecutionState": {
"Operations": [
{
"Id": "exec-0",
"Type": "Execution",
"Status": "STARTED"
}
],
"NextMarker": ""
}
});
let (_log, marker) = parse_inline_operations(&payload);
assert_eq!(marker, None);
}
#[test]
fn parse_inline_operations_keeps_marker_without_operations() {
let payload = serde_json::json!({
"InitialExecutionState": {
"NextMarker": "page-token-1"
}
});
let (log, marker) = parse_inline_operations(&payload);
assert!(!log.has_records());
assert_eq!(marker, Some("page-token-1".to_owned()));
}
fn make_test_step_op(id: &str, result: &str) -> aws_sdk_lambda::types::Operation {
aws_sdk_lambda::types::Operation::builder()
.id(id)
.r#type(aws_sdk_lambda::types::OperationType::Step)
.status(aws_sdk_lambda::types::OperationStatus::Succeeded)
.start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.result(result)
.build(),
)
.build()
.unwrap()
}
fn make_test_execution_op(input: &str) -> aws_sdk_lambda::types::Operation {
aws_sdk_lambda::types::Operation::builder()
.id("execution")
.r#type(aws_sdk_lambda::types::OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(0))
.execution_details(
aws_sdk_lambda::types::ExecutionDetails::builder()
.input_payload(input)
.build(),
)
.build()
.unwrap()
}
#[tokio::test]
async fn resolve_bootstrap_log_paginates_when_marker_present() {
let all_ops = vec![
make_test_execution_op("{\"fetched\":true}"),
make_test_step_op("step-1", "\"r1\""),
make_test_step_op("step-2", "\"r2\""),
];
let client = client::InMemoryExecutionClient::new(all_ops);
let inline_log = engine::CheckpointLog::empty();
let result = client::resolve_bootstrap_log(
&client,
"arn:test",
"token",
client::BootstrapState {
log: inline_log,
input_payload: None,
execution: None,
},
Some("page-2-marker"),
)
.await;
assert!(result.is_ok());
let client::BootstrapState {
log,
input_payload: input,
execution,
} = result.unwrap();
let execution = execution.expect("the fetched Execution operation");
assert_eq!(execution.id(), "execution");
assert_eq!(execution.kind(), Some(OperationKind::Execution));
assert_eq!(execution.status(), Some(OperationStatus::Started));
assert_eq!(
execution.start_timestamp(),
Some(std::time::UNIX_EPOCH),
"the fetched start timestamp"
);
assert_eq!(
input.as_deref(),
Some("{\"fetched\":true}"),
"the input must come from the fetched Execution operation"
);
assert!(
log.get("execution").is_none(),
"the Execution operation must not enter the log"
);
assert!(log.get("step-1").is_some(), "step-1 must be in the log");
assert!(
log.get("step-2").is_some(),
"step-2 must be in the log (from page 2)"
);
let count = *client.get_state_call_count.lock().unwrap();
assert_eq!(
count, 1,
"get_state must be called exactly once when marker is present"
);
}
#[tokio::test]
async fn resolve_bootstrap_log_skips_pagination_when_no_marker() {
let client = client::InMemoryExecutionClient::new(Vec::new());
let inline_log = engine::CheckpointLog::empty();
inline_log.insert(
"existing-op".to_owned(),
engine::CheckpointRecord {
id: "existing-op".to_owned(),
status: engine::CheckpointStatus::Succeeded,
result: Some("\"inline\"".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 1,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
);
let inline_execution = execution_info_from_json(&serde_json::json!({
"Id": "inline-exec",
"Type": "EXECUTION",
"Status": "STARTED",
"StartTimestamp": 4000
}));
assert!(inline_execution.is_some());
let result = client::resolve_bootstrap_log(
&client,
"arn:test",
"token",
client::BootstrapState {
log: inline_log,
input_payload: Some("\"inline-input\"".to_owned()),
execution: inline_execution,
},
None,
)
.await;
assert!(result.is_ok());
let client::BootstrapState {
log,
input_payload: input,
execution,
} = result.unwrap();
let execution = execution.expect("the inline Execution operation");
assert_eq!(execution.id(), "inline-exec");
assert_eq!(
execution.start_timestamp(),
Some(std::time::UNIX_EPOCH + Duration::from_secs(4))
);
assert_eq!(input.as_deref(), Some("\"inline-input\""));
assert!(
log.get("existing-op").is_some(),
"inline op must be preserved"
);
let count = *client.get_state_call_count.lock().unwrap();
assert_eq!(
count, 0,
"get_state must not be called when no marker is present"
);
}
#[test]
fn sdk_config_measurably_alters_client_construction() {
let sdk_config = SdkConfig::builder()
.behavior_version(aws_config::BehaviorVersion::latest())
.region(aws_sdk_lambda::config::Region::new("eu-west-1"))
.build();
let client = base_lambda_client_from_options(Some(sdk_config), None)
.expect("sdk_config must yield a client");
assert_eq!(
client.config().region().map(ToString::to_string),
Some("eu-west-1".to_owned()),
"the supplied sdk_config's region must flow into the built client"
);
}
#[test]
fn supplied_lambda_client_is_the_one_used() {
let conf = aws_sdk_lambda::config::Config::builder()
.behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
.region(aws_sdk_lambda::config::Region::new("ap-south-1"))
.build();
let supplied = aws_sdk_lambda::Client::from_conf(conf);
let resolved = base_lambda_client_from_options(None, Some(supplied))
.expect("lambda_client must be returned");
assert_eq!(
resolved.config().region().map(ToString::to_string),
Some("ap-south-1".to_owned()),
"the supplied lambda_client must be used verbatim, not replaced"
);
}
#[test]
fn no_options_defers_client_to_ambient_default() {
assert!(base_lambda_client_from_options(None, None).is_none());
}
#[tokio::test]
async fn client_provider_reuses_preset_across_invocations() {
use crate::client::InMemoryExecutionClient;
use std::sync::Arc as StdArc;
let preset: StdArc<dyn ExecutionClient> =
StdArc::new(InMemoryExecutionClient::new(Vec::new()));
let provider = ClientProvider::new(Some(StdArc::clone(&preset)));
let first = provider.get().await;
let second = provider.get().await;
assert!(
StdArc::ptr_eq(&first, &second),
"the client must be reused, not rebuilt, across invocations"
);
assert!(
StdArc::ptr_eq(&first, &preset),
"the reused client must be exactly the one supplied via Options"
);
}
#[test]
fn parse_envelope_valid_payload() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-west-2:123456789012:function:test",
"CheckpointToken": "tok-abc",
"InitialExecutionState": { "Operations": [] }
});
let result = parse_envelope(&payload);
assert!(result.is_ok());
let envelope = result.unwrap().expect("envelope should be Some");
assert_eq!(
envelope.execution_arn,
"arn:aws:lambda:us-west-2:123456789012:function:test"
);
assert_eq!(envelope.checkpoint_token, "tok-abc");
}
#[test]
fn parse_envelope_missing_arn_errors() {
let payload = serde_json::json!({
"CheckpointToken": "tok-abc",
"InitialExecutionState": { "Operations": [] }
});
let err = parse_envelope(&payload).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("DurableExecutionArn") && msg.contains("missing"),
"error should name the missing field, got: {msg}"
);
}
#[test]
fn parse_envelope_missing_token_errors() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"InitialExecutionState": { "Operations": [] }
});
let err = parse_envelope(&payload).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("CheckpointToken") && msg.contains("missing"),
"error should name the missing field, got: {msg}"
);
}
#[test]
fn parse_envelope_arn_wrong_type_errors() {
let payload = serde_json::json!({
"DurableExecutionArn": 12345,
"CheckpointToken": "tok-abc",
"InitialExecutionState": { "Operations": [] }
});
let err = parse_envelope(&payload).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("DurableExecutionArn") && msg.contains("not a string"),
"error should note the type mismatch, got: {msg}"
);
}
#[test]
fn parse_envelope_token_wrong_type_errors() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": ["not", "a", "string"],
"InitialExecutionState": { "Operations": [] }
});
let err = parse_envelope(&payload).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("CheckpointToken") && msg.contains("not a string"),
"error should note the type mismatch, got: {msg}"
);
}
#[test]
fn envelope_updated_operation_ids_are_read() {
let ids = |extra: serde_json::Value| {
let mut payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
});
if let (Some(map), serde_json::Value::Object(extra)) = (payload.as_object_mut(), extra)
{
map.extend(extra);
}
parse_envelope(&payload)
.unwrap()
.unwrap()
.updated_operation_ids
};
assert_eq!(
ids(serde_json::json!({ "UpdatedOperationIds": ["a", 7, "b"] })),
["a", "b"]
);
assert_eq!(ids(serde_json::json!({})), Vec::<String>::new());
assert_eq!(
ids(serde_json::json!({ "UpdatedOperationIds": "a" })),
Vec::<String>::new()
);
}
#[test]
fn inline_operation_parent_and_timestamps() {
let op = serde_json::json!({
"Id": "s",
"Type": "STEP",
"Status": "STARTED",
"ParentId": "p",
"StartTimestamp": 1500,
"EndTimestamp": "later"
});
let (_, record) = parse_single_operation(&op).unwrap();
assert_eq!(record.parent_id.as_deref(), Some("p"));
assert_eq!(
record.start_timestamp,
Some(std::time::UNIX_EPOCH + Duration::from_millis(1500))
);
assert_eq!(record.end_timestamp, None);
}
#[test]
fn unknown_type_and_status_keep_wire_tokens() {
let op = serde_json::json!({
"Id": "f",
"Type": "FUTURE_OP",
"Status": "FUTURE_STATUS"
});
let (id, record) = parse_single_operation(&op).unwrap();
let info = plugin::OperationInfo::from_record(&id, &record);
assert_eq!(info.kind(), None);
assert_eq!(info.status(), None);
assert_eq!(info.type_str(), Some("FUTURE_OP"));
assert_eq!(info.status_str(), Some("FUTURE_STATUS"));
}
#[tokio::test]
async fn bootstrap_state_reads_execution() {
let inline = execution_info_from_json(&serde_json::json!({
"Id": "exec",
"Type": "EXECUTION",
"Status": "STARTED",
"StartTimestamp": 2000
}))
.expect("an inline EXECUTION operation");
assert_eq!(
inline.start_timestamp(),
Some(std::time::UNIX_EPOCH + Duration::from_secs(2))
);
let fetched_execution = aws_sdk_lambda::types::Operation::builder()
.id("exec")
.r#type(aws_sdk_lambda::types::OperationType::Execution)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_sdk_lambda::primitives::DateTime::from_secs(3))
.build()
.unwrap();
let client = client::InMemoryExecutionClient::new(vec![
fetched_execution,
make_test_step_op("s1", "\"r\""),
]);
let state = client::resolve_bootstrap_log(
&client,
"arn:test",
"token",
client::BootstrapState {
log: engine::CheckpointLog::empty(),
input_payload: None,
execution: None,
},
Some("m"),
)
.await
.unwrap();
assert_eq!(
state.execution.and_then(|e| e.start_timestamp()),
Some(std::time::UNIX_EPOCH + Duration::from_secs(3))
);
}
#[test]
fn execution_info_from_json_rejects_other_operations() {
assert!(
execution_info_from_json(&serde_json::json!({ "Id": "s", "Type": "STEP" })).is_none()
);
assert!(execution_info_from_json(&serde_json::json!({ "Type": "EXECUTION" })).is_none());
assert!(execution_info_from_json(&serde_json::json!({ "Id": "e" })).is_none());
let info = execution_info_from_json(&serde_json::json!({
"Id": "e",
"Type": "Execution",
"ParentId": "p",
"EndTimestamp": 5000
}))
.expect("the PascalCase spelling is an EXECUTION operation");
assert_eq!(info.status(), Some(OperationStatus::Started));
assert_eq!(info.parent_id(), Some("p"));
assert_eq!(
info.end_timestamp(),
Some(std::time::UNIX_EPOCH + Duration::from_secs(5))
);
}
#[derive(Clone, Default)]
struct RecordingFactory(std::sync::Arc<std::sync::Mutex<Vec<plugin::InvocationInfo>>>);
impl plugin::PluginFactory for RecordingFactory {
fn create_plugin(
&self,
info: &plugin::InvocationInfo,
) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
self.0.lock().unwrap().push(info.clone());
Ok(None)
}
}
async fn infos_for(
payload: serde_json::Value,
lambda_ctx: lambda_runtime::Context,
) -> Vec<plugin::InvocationInfo> {
let factory = RecordingFactory::default();
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(factory.clone())
.build()
.unwrap();
let service = wrap(
|_event: serde_json::Value, _ctx: DurableContext| async move {
Ok::<serde_json::Value, BoxError>(serde_json::Value::Null)
},
options,
);
service(lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_ctx,
))
.await
.unwrap();
factory.0.lock().unwrap().clone()
}
fn envelope_with(operations: &serde_json::Value) -> serde_json::Value {
serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": { "Operations": operations }
})
}
#[tokio::test]
async fn operations_map_contains_execution() {
let payload = envelope_with(&serde_json::json!([
{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"StartTimestamp": 7000,
"ExecutionDetails": { "InputPayload": "{}" }
},
{ "Id": "s1", "Type": "STEP", "Status": "SUCCEEDED", "ParentId": "exec-1" },
{ "Id": "c1", "Type": "CONTEXT", "Status": "STARTED" }
]));
let infos = infos_for(payload, lambda_runtime::Context::default()).await;
assert_eq!(infos.len(), 1);
let operations = infos.first().unwrap().operations();
let exec = operations.get("exec-1").expect("the EXECUTION operation");
assert_eq!(exec.kind(), Some(OperationKind::Execution));
assert_eq!(exec.status(), Some(OperationStatus::Started));
assert_eq!(exec.status_str(), Some("STARTED"));
assert_eq!(
exec.start_timestamp(),
Some(std::time::UNIX_EPOCH + Duration::from_secs(7))
);
assert_eq!(
operations
.get("s1")
.and_then(plugin::OperationInfo::parent_id),
Some("exec-1")
);
assert_eq!(
operations.get("c1").map(plugin::OperationInfo::parent_id),
Some(None)
);
assert_eq!(
infos.first().unwrap().execution_start_timestamp(),
Some(std::time::UNIX_EPOCH + Duration::from_secs(7))
);
assert!(!infos.first().unwrap().is_first_invocation());
}
#[tokio::test]
async fn invocation_info_reads_lambda_context() {
let payload = || {
envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{\"k\":1}" }
}]))
};
let mut lambda_ctx = lambda_runtime::Context::default();
lambda_ctx.request_id = "r-1".to_owned();
lambda_ctx.xray_trace_id = Some("Root=1-abc".to_owned());
let infos = infos_for(payload(), lambda_ctx).await;
let info = infos.first().unwrap();
assert_eq!(info.request_id(), "r-1");
assert_eq!(info.x_ray_trace_id(), Some("Root=1-abc"));
assert_eq!(info.execution_arn(), "arn:test");
assert_eq!(info.execution_input(), "{\"k\":1}");
assert!(info.is_first_invocation());
let mut lambda_ctx = lambda_runtime::Context::default();
lambda_ctx.xray_trace_id = None;
let infos = infos_for(payload(), lambda_ctx).await;
assert_eq!(infos.first().unwrap().x_ray_trace_id(), None);
}
#[tokio::test]
async fn invocation_info_defaults_input() {
let payload = envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED"
}]));
let infos = infos_for(payload, lambda_runtime::Context::default()).await;
assert_eq!(infos.first().unwrap().execution_input(), "{}");
}
struct PanickingDropPlugin;
impl plugin::Plugin for PanickingDropPlugin {}
impl Drop for PanickingDropPlugin {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("plugin drop");
}
}
#[derive(Debug)]
struct PanickingDropError;
impl std::fmt::Display for PanickingDropError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("drop err")
}
}
impl std::error::Error for PanickingDropError {}
impl Drop for PanickingDropError {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("error drop");
}
}
struct PanickingDropFactory {
failing: bool,
}
impl plugin::PluginFactory for PanickingDropFactory {
fn create_plugin(
&self,
_info: &plugin::InvocationInfo,
) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
if self.failing {
Err(Box::new(PanickingDropError))
} else {
Ok(Some(Box::new(PanickingDropPlugin)))
}
}
}
fn service_with_factory<F, Fut>(
factory: PanickingDropFactory,
handler: F,
) -> (
impl Fn(lambda_runtime::LambdaEvent<InvocationPayload>) -> BoxedInvocationFuture,
lambda_runtime::LambdaEvent<InvocationPayload>,
)
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<serde_json::Value, BoxError>> + Send,
{
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(factory)
.build()
.unwrap();
let payload = envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}]));
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_runtime::Context::default(),
);
(wrap(handler, options), event)
}
#[tokio::test]
async fn plugin_drop_panic_keeps_successful_outcome() {
let (service, event) = service_with_factory(
PanickingDropFactory { failing: false },
|_event, _ctx| async move { Ok(serde_json::json!("done")) },
);
let json = service(event).await.unwrap().0;
assert_eq!(
json.get("Status"),
Some(&serde_json::json!("SUCCEEDED")),
"{json}"
);
assert_eq!(
json.get("Result"),
Some(&serde_json::json!("\"done\"")),
"{json}"
);
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn plugin_drop_panic_while_handler_panics_resumes_handler_panic() {
#[expect(clippy::panic)] fn boom() -> Result<serde_json::Value, BoxError> {
panic!("root boom")
}
let (service, event) = service_with_factory(
PanickingDropFactory { failing: false },
|_event, _ctx| async move { boom() },
);
let join_err = tokio::spawn(service(event))
.await
.expect_err("the handler panic leaves the service");
assert!(join_err.is_panic(), "a panic, not a cancellation");
let payload = join_err.into_panic();
assert_eq!(payload.downcast_ref::<&str>(), Some(&"root boom"));
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn factory_error_destructor_panic_fails_the_invocation() {
let (service, event) = service_with_factory(
PanickingDropFactory { failing: true },
|_event, _ctx| async move { Ok(serde_json::Value::Null) },
);
let join_err = tokio::spawn(service(event))
.await
.expect_err("the destructor panic leaves the service");
assert!(join_err.is_panic(), "a panic, not a cancellation");
let payload = join_err.into_panic();
assert_eq!(payload.downcast_ref::<&str>(), Some(&"error drop"));
}
use plugin::test_support::{self as hooks, Hooks};
#[tokio::test(start_paused = true)]
#[expect(clippy::panic)] async fn end_hook_cut_off_before_deadline() {
for _attempt in 0..5 {
let (buffer, _guard) = hooks::capture();
let plugin = Hooks::default().on_end(|_| Box::pin(std::future::pending()));
let (service, event) = hooks::service(
vec![plugin],
hooks::deadline_in(1_000),
|_e, _ctx| async move { Ok(serde_json::json!("ok")) },
);
let json = service(event).await.unwrap().0;
assert_eq!(json.get("Status"), Some(&serde_json::json!("SUCCEEDED")));
let warnings = hooks::warnings(&buffer);
let [_] = hooks::with_message(
&warnings,
"plugin on_invocation_end hooks did not finish before the invocation deadline; \
the response returns without them",
)[..] else {
continue;
};
return;
}
panic!("no attempt captured the warning");
}
#[tokio::test(start_paused = true)]
async fn end_hook_unbounded_without_deadline() {
let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let plugin = {
let flag = std::sync::Arc::clone(&flag);
Hooks::default().on_end(move |_| {
let flag = std::sync::Arc::clone(&flag);
Box::pin(async move {
tokio::time::sleep(Duration::from_millis(100)).await;
flag.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
})
})
};
let (service, event) = hooks::service(vec![plugin], 0, |_e, _ctx| async move {
Ok(serde_json::Value::Null)
});
service(event).await.unwrap();
assert!(flag.load(std::sync::atomic::Ordering::SeqCst));
}
#[tokio::test]
async fn end_info_projects_operations_once() {
async fn thousand_steps(
_event: serde_json::Value,
ctx: DurableContext,
) -> Result<serde_json::Value, BoxError> {
for i in 1..=1_000_u32 {
ctx.step(move |_| async move { Ok(i) }).await?;
}
Ok(serde_json::Value::Null)
}
let recorded = || {
(1..=1_000_u32)
.map(|i| {
serde_json::json!({
"Id": engine::compute_wire_id_public(&i.to_string()),
"Type": "STEP",
"SubType": "Step",
"Status": "SUCCEEDED",
"StepDetails": { "Result": i.to_string() }
})
})
.collect::<Vec<_>>()
};
let mut growths = Vec::new();
for count in [1, 3] {
let plugins: Vec<Hooks> = (0..count).map(|_| Hooks::default()).collect();
let (service, event) =
hooks::service_with(plugins.clone(), 0, recorded(), thousand_steps);
let before = engine::projections();
service(event).await.unwrap();
growths.push(engine::projections() - before);
for plugin in &plugins {
let ends = plugin.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info per plugin, got {}", ends.len());
};
assert_eq!(end.operations().len(), 1_001);
}
}
let [one, three] = growths.as_slice() else {
unreachable!("two runs");
};
assert_eq!(one, three);
}
fn json_str<'a>(json: &'a serde_json::Value, pointer: &str) -> Option<&'a str> {
json.pointer(pointer).and_then(serde_json::Value::as_str)
}
async fn finish_with_failing_flush<R>(
root: impl FnOnce(DurableContext) -> R,
) -> (
serde_json::Value,
plugin::InvocationEndInfo,
Vec<aws_sdk_lambda::types::OperationUpdate>,
)
where
R: Future<Output = Result<String, WireError>> + Send + 'static,
{
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(client::TestResponse::NonRetryableError(
"flush down".to_owned(),
));
let hook = Hooks::default();
let plugins = hooks::plugins_of(hook.clone());
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
std::sync::Arc::new(engine::CheckpointLog::empty()),
std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>,
"t0".to_owned(),
Duration::from_secs(60),
0,
None,
);
let update = aws_sdk_lambda::types::OperationUpdate::builder()
.id("s")
.name("s")
.r#type(aws_sdk_lambda::types::OperationType::Step)
.action(aws_sdk_lambda::types::OperationAction::Succeed)
.payload("1")
.build()
.expect("all required fields set");
ctx.buffer_unflushed(vec![update]);
let end_log = engine::CheckpointLog::empty();
let finished = finish_invocation(
root(ctx.clone()),
task::TaskContext::current(),
std::sync::Arc::clone(ctx.task_ownership()),
std::sync::Arc::clone(ctx.coordinator()),
&ctx,
None,
Some(&plugins),
&end_log,
)
.await;
let Finished::Respond(response, _late) = finished else {
unreachable!("the invocation answers");
};
let ends = hook.ends();
let [end] = ends.as_slice() else {
unreachable!("one end hook, got {}", ends.len());
};
(
response.expect("an envelope").0,
end.clone(),
client.recorded_updates(),
)
}
#[tokio::test]
async fn flush_failure_decides_over_handler_error() {
let (json, end, updates) = finish_with_failing_flush(|_ctx| async {
Err(WireError::new(Some("Error"), Some("boom")))
})
.await;
assert_eq!(json_str(&json, "/Status"), Some("FAILED"), "{json}");
assert_eq!(
json_str(&json, "/Error/ErrorType"),
Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
assert_eq!(end.status(), InvocationStatus::Failed);
assert_eq!(
end.execution_error().and_then(WireError::error_type),
Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
assert_eq!(updates.len(), 1, "only the flushed SUCCEED: {updates:?}");
assert!(
updates
.iter()
.all(|u| u.action() != &aws_sdk_lambda::types::OperationAction::Fail),
"{updates:?}"
);
}
#[tokio::test]
async fn flush_failure_after_a_winning_fatal_keeps_the_fatal() {
let (json, end, _updates) = finish_with_failing_flush(|ctx| async move {
ctx.coordinator()
.record_fatal("UsageError".to_owned(), "misuse".to_owned());
std::future::pending().await
})
.await;
assert_eq!(json_str(&json, "/Status"), Some("FAILED"), "{json}");
assert_eq!(json_str(&json, "/Error/ErrorType"), Some("UsageError"));
assert_eq!(
end.execution_error().and_then(WireError::error_type),
Some("UsageError")
);
}
#[test]
fn late_panics_fail_only_a_response_without_a_fatal_error() {
use crate::plugin::LatePanics;
let ok = Ok(InvocationResponse(serde_json::Value::Null));
let err = Err(lambda_runtime::Error::from("fault"));
assert_eq!(late_panics(Some(&ok), false), LatePanics::Fail);
assert_eq!(late_panics(Some(&err), false), LatePanics::Discard);
assert_eq!(late_panics(None, false), LatePanics::Discard);
assert_eq!(late_panics(Some(&ok), true), LatePanics::Discard);
}
#[expect(clippy::disallowed_methods)]
async fn panic_payload_of<Fut>(future: Fut) -> Box<dyn std::any::Any + Send>
where
Fut: Future + Send + 'static,
Fut::Output: Send,
{
match tokio::spawn(future).await {
Ok(_) => unreachable!("the service must panic"),
Err(join_err) => join_err.into_panic(),
}
}
#[tokio::test]
#[expect(clippy::panic)] async fn end_hook_retrying_after_panic() {
let messages = [
"boom",
"boom",
"the invocation panicked with a payload that is not a string",
];
for (case, message) in messages.into_iter().enumerate() {
let plugin = Hooks::default();
let (service, event) =
hooks::service(vec![plugin.clone()], 0, move |_e, _ctx| async move {
match case {
0 => panic!("boom"),
1 => panic!("{}", String::from("boom")),
_ => std::panic::panic_any(42_u32),
}
});
let resumed = panic_payload_of(service(event)).await;
let original = match case {
0 => resumed.downcast_ref::<&str>() == Some(&"boom"),
1 => resumed.downcast_ref::<String>().map(String::as_str) == Some("boom"),
_ => resumed.downcast_ref::<u32>() == Some(&42),
};
assert!(original, "case {case}: the original payload leaves the SDK");
let ends = plugin.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Retrying);
assert_eq!(end.execution_result(), None);
let error = end.execution_error().expect("a panic end has an error");
assert_eq!(error.error_type(), None);
assert_eq!(error.error_message(), Some(message));
}
}
#[tokio::test]
#[expect(clippy::panic)] async fn end_hook_after_panic_runs_after_cleanup() {
struct Guard(std::sync::Arc<std::sync::atomic::AtomicBool>);
impl Drop for Guard {
fn drop(&mut self) {
self.0.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
let dropped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let seen = std::sync::Arc::new(std::sync::Mutex::new(None));
let plugin = {
let dropped = std::sync::Arc::clone(&dropped);
let seen = std::sync::Arc::clone(&seen);
Hooks::default().on_end(move |_| {
*seen.lock().unwrap() = Some(dropped.load(std::sync::atomic::Ordering::SeqCst));
Box::pin(std::future::ready(Ok(())))
})
};
let (service, event) = hooks::service(vec![plugin], 0, move |_e, ctx| {
let dropped = std::sync::Arc::clone(&dropped);
async move {
let _held = ctx
.step(move |_| {
let guard = Guard(std::sync::Arc::clone(&dropped));
async move {
let _guard = guard;
std::future::pending::<()>().await;
Ok(0_u32)
}
})
.spawn();
tokio::time::sleep(Duration::from_millis(10)).await;
panic!("boom")
}
});
let payload = panic_payload_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
assert_eq!(*seen.lock().unwrap(), Some(true));
}
#[tokio::test]
#[expect(clippy::panic)] async fn plugins_drop_before_the_panic_resumes() {
struct Probe(std::sync::Arc<std::sync::Mutex<Option<bool>>>);
impl plugin::Plugin for Probe {}
impl Drop for Probe {
fn drop(&mut self) {
*self.0.lock().unwrap() = Some(std::thread::panicking());
}
}
struct ProbeFactory(std::sync::Arc<std::sync::Mutex<Option<bool>>>);
impl plugin::PluginFactory for ProbeFactory {
fn create_plugin(
&self,
_info: &plugin::InvocationInfo,
) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
Ok(Some(Box::new(Probe(std::sync::Arc::clone(&self.0)))))
}
}
let unwinding = std::sync::Arc::new(std::sync::Mutex::new(None));
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(ProbeFactory(std::sync::Arc::clone(&unwinding)))
.build()
.unwrap();
let service = wrap(
|_e: serde_json::Value, _ctx: DurableContext| async move {
panic!("boom");
#[expect(unreachable_code)] Ok::<serde_json::Value, BoxError>(serde_json::Value::Null)
},
options,
);
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}]))),
lambda_runtime::Context::default(),
);
let payload = panic_payload_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
assert_eq!(*unwinding.lock().unwrap(), Some(false));
}
#[tokio::test]
#[expect(clippy::panic)] async fn panicking_end_hook_keeps_the_original_panic() {
for _attempt in 0..5 {
let (buffer, _guard) = hooks::capture();
let faulty = Hooks::default().named("Faulty").on_end(|_| panic!("hook"));
let healthy = Hooks::default();
let (service, event) =
hooks::service(vec![faulty, healthy.clone()], 0, |_e, _ctx| async move {
panic!("boom")
});
let payload = panic_payload_of(service(event)).await;
assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
let ends = healthy.ends();
let [end] = ends.as_slice() else {
unreachable!("one end info, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Retrying);
let warnings = hooks::warnings(&buffer);
let [warning] =
hooks::with_message(&warnings, "plugin hook panicked; the panic is ignored")[..]
else {
continue;
};
assert!(
hooks::field(warning, "plugin").ends_with("Faulty (plugins[0])"),
"{warning}"
);
assert_eq!(hooks::field(warning, "hook"), "on_invocation_end");
return;
}
panic!("no attempt captured the warning");
}
#[derive(Clone, Copy, PartialEq)]
enum ClaimAt {
AfterNext,
OperationStart,
}
type ContextSlot = std::sync::Arc<std::sync::Mutex<Option<DurableContext>>>;
#[derive(Clone)]
struct ClaimPlugin {
at: ClaimAt,
slot: ContextSlot,
claims: std::sync::Arc<std::sync::atomic::AtomicUsize>,
armed: std::sync::Arc<std::sync::atomic::AtomicBool>,
}
impl ClaimPlugin {
fn new(at: ClaimAt) -> Self {
Self {
at,
slot: std::sync::Arc::default(),
claims: std::sync::Arc::default(),
armed: std::sync::Arc::default(),
}
}
fn claims(&self) -> usize {
self.claims.load(std::sync::atomic::Ordering::SeqCst)
}
fn claim_once(&self, ctx: &DurableContext) {
if self
.claims
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
== 0
{
drop(ctx.wait(Duration::from_secs(1)));
}
}
}
impl plugin::Plugin for ClaimPlugin {
fn on_operation_start(&self, _info: &plugin::OperationInfo) -> plugin::HookWork {
if self.at == ClaimAt::OperationStart
&& let Some(ctx) = self.slot.lock().unwrap().clone()
{
self.claim_once(&ctx);
}
Ok(None)
}
fn wrap_invocation(&self, _info: &plugin::InvocationInfo, next: &mut dyn FnMut()) {
next();
let ctx = self.slot.lock().unwrap().clone();
if self.at == ClaimAt::AfterNext
&& let Some(ctx) = ctx
&& ctx.coordinator().phase() != coordinator::Phase::Running
&& self.armed.load(std::sync::atomic::Ordering::SeqCst)
{
self.claim_once(&ctx);
}
}
}
impl plugin::PluginFactory for ClaimPlugin {
fn create_plugin(
&self,
_info: &plugin::InvocationInfo,
) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
#[derive(Debug)]
struct LateDropError;
impl std::fmt::Display for LateDropError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("late drop error")
}
}
impl std::error::Error for LateDropError {}
impl Drop for LateDropError {
#[expect(clippy::panic)] fn drop(&mut self) {
panic!("late drop");
}
}
fn first_invocation_event() -> lambda_runtime::LambdaEvent<InvocationPayload> {
lambda_runtime::LambdaEvent::new(
InvocationPayload(envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
}]))),
lambda_runtime::Context::default(),
)
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn late_panic_after_post_drive_claim_fails_the_invocation() {
let (buffer, _guard) = hooks::capture();
let claimer = ClaimPlugin::new(ClaimAt::AfterNext);
let claims_at_end = std::sync::Arc::new(std::sync::Mutex::new(None));
let failing = {
let claimer = claimer.clone();
let claims_at_end = std::sync::Arc::clone(&claims_at_end);
Hooks::default().on_end(move |_| {
*claims_at_end.lock().unwrap() = Some(claimer.claims());
Box::pin(async { Err(Box::new(LateDropError) as BoxError) })
})
};
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(claimer.clone())
.plugin(failing)
.build()
.unwrap();
let slot = std::sync::Arc::clone(&claimer.slot);
let service = wrap(
move |_e: serde_json::Value, ctx: DurableContext| {
*slot.lock().unwrap() = Some(ctx);
async { Ok::<_, BoxError>(serde_json::json!("done")) }
},
options,
);
let joined = tokio::spawn(test_hooks::PENDING_AFTER_DRIVE.scope(
std::sync::Arc::clone(&claimer.armed),
service(first_invocation_event()),
))
.await;
let payload = match joined {
Ok(response) => unreachable!(
"the late panic must fail the invocation: {:?}",
response.map(|r| r.0)
),
Err(join_err) => join_err.into_panic(),
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"late drop"));
assert_eq!(
*claims_at_end.lock().unwrap(),
Some(1),
"the claim ran after next() in the post-drive pending poll, before the end hooks"
);
let ctx = claimer
.slot
.lock()
.unwrap()
.take()
.expect("the handler ran");
let fatal = ctx
.coordinator()
.fatal_error()
.map(|fatal| fatal.error_message);
assert!(
fatal
.as_deref()
.is_some_and(|m| m.contains("inside plugin code")),
"the claim records the context usage error: {fatal:?}"
);
let warnings = hooks::warnings(&buffer);
assert_eq!(
hooks::with_message(&warnings, "plugin hook failed; the error is ignored").len(),
1,
"{warnings:?}"
);
assert_eq!(warnings.len(), 1, "no claim WARN: {warnings:?}");
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn fatal_after_driver_arbitration_keeps_the_late_panic() {
let failing = Hooks::default()
.on_end(|_| Box::pin(async { Err(Box::new(LateDropError) as BoxError) }));
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(failing)
.build()
.unwrap();
let slot: ContextSlot = std::sync::Arc::default();
let service = {
let slot = std::sync::Arc::clone(&slot);
wrap(
move |_e: serde_json::Value, ctx: DurableContext| {
*slot.lock().unwrap() = Some(ctx);
async { Ok::<_, BoxError>(serde_json::json!("done")) }
},
options,
)
};
let hook: std::sync::Arc<dyn Fn(&coordinator::InvocationCoordinator) + Send + Sync> = {
let slot = std::sync::Arc::clone(&slot);
std::sync::Arc::new(move |_coordinator| {
let ctx = slot.lock().unwrap().clone().expect("the handler ran");
plugin::call_plugin(|| drop(ctx.wait(Duration::from_secs(1))))
.expect("the claim does not panic");
})
};
let joined = tokio::spawn(
driver::test_hooks::AFTER_ARBITRATION.scope(hook, service(first_invocation_event())),
)
.await;
let payload = match joined {
Ok(response) => unreachable!(
"the late panic must fail the invocation: {:?}",
response.map(|r| r.0)
),
Err(join_err) => join_err.into_panic(),
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"late drop"));
let ctx = slot.lock().unwrap().take().expect("the handler ran");
let fatal = ctx
.coordinator()
.fatal_error()
.map(|fatal| fatal.error_message);
assert!(
fatal
.as_deref()
.is_some_and(|m| m.contains("inside plugin code")),
"the claim after the arbitration records the context usage error: {fatal:?}"
);
}
#[tokio::test]
async fn claim_in_end_hook_leaves_response_unchanged() {
let slot: ContextSlot = std::sync::Arc::default();
let kept: ContextSlot = std::sync::Arc::default();
let plugin = {
let slot = std::sync::Arc::clone(&slot);
Hooks::default().on_end(move |_| {
if let Some(ctx) = slot.lock().unwrap().as_ref() {
drop(ctx.wait(Duration::from_secs(1)));
}
Box::pin(std::future::ready(Ok(())))
})
};
let handler_slot = std::sync::Arc::clone(&slot);
let handler_kept = std::sync::Arc::clone(&kept);
let (service, event) = hooks::service(vec![plugin], 0, move |_e, ctx| {
*handler_slot.lock().unwrap() = Some(ctx.clone());
*handler_kept.lock().unwrap() = Some(ctx);
async move { Ok(serde_json::json!("done")) }
});
let json = service(event).await.unwrap().0;
let ctx = kept.lock().unwrap().take().expect("the handler ran");
drop(slot.lock().unwrap().take());
let fatal = ctx
.coordinator()
.fatal_error()
.map(|fatal| fatal.error_message);
assert!(
fatal
.as_deref()
.is_some_and(|m| m.contains("inside plugin code")),
"the end-hook claim records the context usage error: {fatal:?}"
);
assert_eq!(
json.get("Status"),
Some(&serde_json::json!("SUCCEEDED")),
"{json}"
);
assert_eq!(
json.get("Result"),
Some(&serde_json::json!("\"done\"")),
"{json}"
);
}
#[tokio::test]
async fn a_claim_while_the_handler_runs_fails_the_execution() {
let claimer = ClaimPlugin::new(ClaimAt::OperationStart);
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(claimer.clone())
.build()
.unwrap();
let slot = std::sync::Arc::clone(&claimer.slot);
let service = wrap(
move |_e: serde_json::Value, ctx: DurableContext| {
*slot.lock().unwrap() = Some(ctx.clone());
async move {
let value = ctx.step(|_| async { Ok(1_u32) }).name("s").await?;
Ok::<_, BoxError>(serde_json::json!(value))
}
},
options,
);
let json = service(first_invocation_event()).await.unwrap().0;
drop(claimer.slot.lock().unwrap().take());
assert_eq!(claimer.claims(), 1, "on_operation_start claimed once");
assert_eq!(
json.get("Status"),
Some(&serde_json::json!("FAILED")),
"{json}"
);
assert!(json.to_string().contains("inside plugin code"), "{json}");
}
type Log = std::sync::Arc<std::sync::Mutex<Vec<String>>>;
struct InvocationWrapLog(Log);
impl plugin::Plugin for InvocationWrapLog {
fn wrap_invocation(&self, _info: &plugin::InvocationInfo, next: &mut dyn FnMut()) {
self.0.lock().unwrap().push("inv enter".to_owned());
next();
self.0.lock().unwrap().push("inv exit".to_owned());
}
}
impl plugin::PluginFactory for InvocationWrapLog {
fn create_plugin(
&self,
_info: &plugin::InvocationInfo,
) -> Result<Option<Box<dyn plugin::Plugin>>, BoxError> {
Ok(Some(Box::new(Self(std::sync::Arc::clone(&self.0)))))
}
}
struct PanicsOnDrop(Log);
impl Drop for PanicsOnDrop {
#[expect(clippy::panic)] fn drop(&mut self) {
self.0.lock().unwrap().push("guard drop".to_owned());
panic!("guard drop");
}
}
#[tokio::test]
async fn pending_invocation_future_destructor_panic() {
let log: Log = std::sync::Arc::default();
let reached = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let held = std::sync::Arc::new(test_hooks::HeldGuard {
reached: std::sync::Arc::clone(&reached),
make: Box::new({
let log = std::sync::Arc::clone(&log);
move || Box::new(PanicsOnDrop(std::sync::Arc::clone(&log))) as Box<dyn Send>
}),
});
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(InvocationWrapLog(std::sync::Arc::clone(&log)))
.build()
.unwrap();
let service = wrap(
|_e: serde_json::Value, _ctx: DurableContext| async {
Ok::<_, BoxError>(serde_json::json!("done"))
},
options,
);
let mut invocation = Box::pin(
test_hooks::PENDING_HOLDING_GUARD.scope(held, service(first_invocation_event())),
);
tokio::time::timeout(
Duration::from_secs(5),
std::future::poll_fn(|cx| {
assert!(
invocation.as_mut().poll(cx).is_pending(),
"the invocation stays pending at the pending point"
);
if reached.load(std::sync::atomic::Ordering::SeqCst) {
std::task::Poll::Ready(())
} else {
std::task::Poll::Pending
}
}),
)
.await
.expect("the post-start future reaches the pending point within 5 s");
log.lock().unwrap().push("dropping".to_owned());
let dropped = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(invocation)));
let payload = dropped.expect_err("the destructor panic propagates");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"guard drop"));
let log = log.lock().unwrap().clone();
let at = log.iter().position(|l| l == "dropping").expect("logged");
assert_eq!(
log.get(at..).expect("in range"),
["dropping", "inv enter", "guard drop", "inv exit"],
"{log:?}"
);
}
#[tokio::test]
async fn failed_input_calls_no_factory() {
let factory = RecordingFactory::default();
let options = Options::builder()
.execution_client(std::sync::Arc::new(client::InMemoryExecutionClient::new(
Vec::new(),
)))
.plugin(factory.clone())
.build()
.unwrap();
let service = wrap(
|_event: u32, _ctx: DurableContext| async move { Ok::<u32, BoxError>(0) },
options,
);
let payload = envelope_with(&serde_json::json!([{
"Id": "exec-1",
"Type": "EXECUTION",
"ExecutionDetails": { "InputPayload": "\"not a number\"" }
}]));
let result = service(lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_runtime::Context::default(),
))
.await;
assert!(result.is_err());
assert!(factory.0.lock().unwrap().is_empty());
}
#[test]
fn parse_envelope_no_envelope_returns_none() {
let payload = serde_json::json!({ "order_id": "abc-123" });
let result = parse_envelope(&payload).unwrap();
assert!(result.is_none(), "non-envelope payload should return None");
}
#[test]
fn extract_customer_input_from_valid_envelope() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": {
"Operations": [{
"Id": "execution",
"Type": "EXECUTION",
"ExecutionDetails": {
"InputPayload": "\"hello\""
}
}]
}
});
let input = inline_input_payload(&payload).unwrap().into_payload();
let result: Result<String, _> = extract_customer_input(input.as_deref());
assert_eq!(result.unwrap(), "hello");
}
#[test]
fn extract_customer_input_defaults_to_empty_object() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": {
"Operations": [{ "Id": "execution", "Type": "EXECUTION" }]
}
});
let input = inline_input_payload(&payload).unwrap();
assert_eq!(input, client::PageInput::Execution(None));
let result: serde_json::Value =
extract_customer_input(input.into_payload().as_deref()).unwrap();
assert_eq!(result, serde_json::json!({}));
}
#[test]
fn inline_input_payload_cases() {
let envelope = |ops: serde_json::Value| {
serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": { "Operations": ops }
})
};
let step_only = envelope(serde_json::json!([{ "Id": "s1", "Type": "STEP" }]));
assert_eq!(
inline_input_payload(&step_only).unwrap(),
client::PageInput::NoExecutionOperation
);
let no_operations = serde_json::json!({
"DurableExecutionArn": "arn:test",
"CheckpointToken": "tok",
"InitialExecutionState": { "NextMarker": "m-1" }
});
assert_eq!(
inline_input_payload(&no_operations).unwrap(),
client::PageInput::NoExecutionOperation
);
let later = envelope(serde_json::json!([
{ "Id": "s1", "Type": "STEP", "ExecutionDetails": { "InputPayload": "\"wrong\"" } },
{ "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": "\"right\"" } }
]));
assert_eq!(
inline_input_payload(&later).unwrap(),
client::PageInput::Execution(Some("\"right\"".to_owned()))
);
let null = envelope(serde_json::json!([
{ "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": null } }
]));
assert_eq!(
inline_input_payload(&null).unwrap(),
client::PageInput::Execution(None)
);
let number = envelope(serde_json::json!([
{ "Id": "e", "Type": "EXECUTION", "ExecutionDetails": { "InputPayload": 5 } }
]));
let err = inline_input_payload(&number).unwrap_err();
assert!(
err.to_string().contains("InputPayload is not a string"),
"got: {err}"
);
}
#[test]
fn extract_customer_input_parse_failure_errors() {
let err = extract_customer_input::<serde_json::Value>(Some("not json")).unwrap_err();
assert!(
err.to_string().contains("deserialize customer input"),
"got: {err}"
);
}
struct PagedTransport {
pages: std::collections::HashMap<String, GetStateOutput>,
markers: std::sync::Mutex<Vec<Option<String>>>,
}
impl PagedTransport {
fn new(pages: Vec<(&str, Vec<aws_sdk_lambda::types::Operation>, Option<&str>)>) -> Self {
Self {
pages: pages
.into_iter()
.map(|(marker, ops, next)| {
(
marker.to_owned(),
GetStateOutput::new(ops, next.map(str::to_owned)),
)
})
.collect(),
markers: std::sync::Mutex::new(Vec::new()),
}
}
fn markers(&self) -> Vec<Option<String>> {
self.markers.lock().unwrap().clone()
}
}
impl ExecutionClient for PagedTransport {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_updates: Vec<aws_sdk_lambda::types::OperationUpdate>,
) -> std::pin::Pin<
Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
> {
Box::pin(async {
Ok(CheckpointOutput::new(
Some("token-next".to_owned()),
Vec::new(),
None,
))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
marker: Option<&str>,
) -> std::pin::Pin<
Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
> {
self.markers.lock().unwrap().push(marker.map(str::to_owned));
let page = marker.and_then(|m| self.pages.get(m)).map_or_else(
|| GetStateOutput::new(Vec::new(), None),
|p| GetStateOutput::new(p.operations.clone(), p.next_marker().map(str::to_owned)),
);
Box::pin(async move { Ok(page) })
}
}
async fn invoke_echo_over(
payload: serde_json::Value,
transport: std::sync::Arc<dyn ExecutionClient>,
) -> (
Result<serde_json::Value, lambda_runtime::Error>,
std::sync::Arc<std::sync::atomic::AtomicBool>,
) {
let ran = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let ran_in_handler = std::sync::Arc::clone(&ran);
let options = Options::builder()
.execution_client(transport)
.build()
.expect("options build");
let service = wrap(
move |event: serde_json::Value, _ctx: DurableContext| {
let ran = std::sync::Arc::clone(&ran_in_handler);
async move {
ran.store(true, std::sync::atomic::Ordering::SeqCst);
Ok::<serde_json::Value, BoxError>(event)
}
},
options,
);
let event = lambda_runtime::LambdaEvent::new(
InvocationPayload(payload),
lambda_runtime::Context::default(),
);
(service(event).await.map(|response| response.0), ran)
}
fn bootstrap_envelope(state: &serde_json::Value) -> serde_json::Value {
serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": state
})
}
fn assert_succeeded_with(response: &serde_json::Value, expected: &serde_json::Value) {
assert_eq!(
response.get("Status").and_then(serde_json::Value::as_str),
Some("SUCCEEDED"),
"unexpected response: {response}"
);
let result: serde_json::Value = serde_json::from_str(
response
.get("Result")
.and_then(serde_json::Value::as_str)
.expect("a Result string"),
)
.expect("the Result parses");
assert_eq!(&result, expected, "unexpected response: {response}");
}
#[tokio::test]
async fn absent_input_payload_runs_on_empty_object() {
for details in [
serde_json::json!({}),
serde_json::json!({ "InputPayload": null }),
] {
let payload = bootstrap_envelope(&serde_json::json!({
"Operations": [{
"Id": "execution",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": details
}]
}));
let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
let (response, _) =
invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
assert_succeeded_with(&response.expect("must succeed"), &serde_json::json!({}));
assert!(transport.markers().is_empty(), "no marker, no get_state");
}
}
#[tokio::test]
async fn input_payload_found_after_pagination() {
let payload = bootstrap_envelope(&serde_json::json!({ "NextMarker": "m-1" }));
let transport = std::sync::Arc::new(PagedTransport::new(vec![(
"m-1",
vec![make_test_execution_op("{\"count\":1}")],
None,
)]));
let (response, _) = invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
assert_succeeded_with(
&response.expect("must succeed"),
&serde_json::json!({ "count": 1 }),
);
assert_eq!(transport.markers(), vec![Some("m-1".to_owned())]);
}
#[tokio::test]
async fn paginated_input_matches_first_page() {
let payload = bootstrap_envelope(&serde_json::json!({
"Operations": [{
"Id": "execution",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{\"a\":1}" }
}],
"NextMarker": "m-1"
}));
let transport = std::sync::Arc::new(PagedTransport::new(vec![(
"m-1",
vec![make_test_step_op("s1", "\"r1\"")],
None,
)]));
let (response, _) = invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
assert_succeeded_with(
&response.expect("must succeed"),
&serde_json::json!({ "a": 1 }),
);
assert_eq!(transport.markers(), vec![Some("m-1".to_owned())]);
}
#[tokio::test]
async fn malformed_input_payload_fails_the_invocation() {
for (input, expected) in [
(serde_json::json!("not json"), "deserialize customer input"),
(serde_json::json!(5), "InputPayload is not a string"),
] {
let payload = bootstrap_envelope(&serde_json::json!({
"Operations": [{
"Id": "execution",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": input }
}]
}));
let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
let (response, ran) =
invoke_echo_over(payload, std::sync::Arc::clone(&transport) as _).await;
let err = response.expect_err("must fail");
assert!(err.to_string().contains(expected), "got: {err}");
assert!(
!ran.load(std::sync::atomic::Ordering::SeqCst),
"the handler must not run"
);
assert!(transport.markers().is_empty(), "no marker, no get_state");
}
}
#[tokio::test]
async fn missing_checkpoint_token_fails_before_input_default() {
let payload = serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"InitialExecutionState": {
"Operations": [{
"Id": "execution",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": {}
}]
}
});
let transport = std::sync::Arc::new(PagedTransport::new(Vec::new()));
let (response, ran) = invoke_echo_over(payload, transport).await;
let err = response.expect_err("must fail");
assert!(err.to_string().contains("CheckpointToken"), "got: {err}");
assert!(
!ran.load(std::sync::atomic::Ordering::SeqCst),
"the handler must not run"
);
}
#[test]
fn default_client_sets_connect_and_read_timeouts() {
let config = SdkConfig::builder()
.behavior_version(aws_config::BehaviorVersion::latest())
.build();
let client = default_lambda_client(&config);
let timeouts = client
.config()
.timeout_config()
.expect("the default client has a timeout config");
assert_eq!(timeouts.connect_timeout(), Some(Duration::from_secs(5)));
assert_eq!(timeouts.read_timeout(), Some(Duration::from_secs(50)));
assert_eq!(
timeouts.operation_attempt_timeout(),
Some(Duration::from_secs(55))
);
}
#[test]
fn supplied_config_keeps_its_timeouts() {
use aws_sdk_lambda::config::timeout::TimeoutConfig;
let config = SdkConfig::builder()
.behavior_version(aws_config::BehaviorVersion::latest())
.timeout_config(
TimeoutConfig::builder()
.connect_timeout(Duration::from_secs(1))
.build(),
)
.build();
let client = base_lambda_client_from_options(Some(config), None)
.expect("sdk_config must yield a client");
let timeouts = client.config().timeout_config().expect("a timeout config");
assert_eq!(timeouts.connect_timeout(), Some(Duration::from_secs(1)));
assert_eq!(timeouts.read_timeout(), None);
let conf = aws_sdk_lambda::config::Config::builder()
.behavior_version(aws_sdk_lambda::config::BehaviorVersion::latest())
.timeout_config(
TimeoutConfig::builder()
.read_timeout(Duration::from_secs(7))
.build(),
)
.build();
let supplied = aws_sdk_lambda::Client::from_conf(conf);
let client = base_lambda_client_from_options(None, Some(supplied))
.expect("lambda_client must be returned");
let timeouts = client.config().timeout_config().expect("a timeout config");
assert_eq!(timeouts.read_timeout(), Some(Duration::from_secs(7)));
assert_eq!(timeouts.connect_timeout(), None);
assert_eq!(timeouts.operation_attempt_timeout(), None);
}
fn oversized_envelope() -> serde_json::Value {
serde_json::json!({
"DurableExecutionArn": "arn:aws:lambda:us-east-1:1:function:test",
"CheckpointToken": "token-abc",
"InitialExecutionState": { "Operations": [] }
})
}
async fn run_sized_result(
n: usize,
client: std::sync::Arc<client::InMemoryExecutionClient>,
) -> (
Result<serde_json::Value, lambda_runtime::Error>,
Vec<plugin::InvocationEndInfo>,
) {
let hooks = Hooks::default();
let options = Options::builder()
.execution_client(client)
.plugin(hooks.clone())
.build()
.expect("valid options");
let service = wrap(
move |_event: serde_json::Value, _ctx: DurableContext| async move {
Ok::<String, BoxError>("a".repeat(n - 2))
},
options,
);
let response = service(lambda_runtime::LambdaEvent::new(
InvocationPayload(oversized_envelope()),
lambda_runtime::Context::default(),
))
.await
.map(|response| response.0);
(response, hooks.ends())
}
fn is_sized_result(serialized: &str, n: usize) -> bool {
serialized.len() == n
&& serialized
.strip_prefix('"')
.and_then(|inner| inner.strip_suffix('"'))
.is_some_and(|inner| inner.bytes().all(|b| b == b'a'))
}
fn execution_updates(
client: &client::InMemoryExecutionClient,
) -> Vec<aws_sdk_lambda::types::OperationUpdate> {
client
.recorded_updates()
.into_iter()
.filter(|update| update.r#type() == &aws_sdk_lambda::types::OperationType::Execution)
.collect()
}
#[tokio::test]
async fn oversized_result_envelope_is_empty() {
let n = LAMBDA_RESPONSE_SIZE_LIMIT + 1;
assert_eq!(n, 6_291_407);
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
let response = response.expect("an envelope");
assert_eq!(
response,
serde_json::json!({ "Status": "SUCCEEDED", "Result": "" })
);
let updates = execution_updates(&client);
let [update] = updates.as_slice() else {
unreachable!("one EXECUTION update, got {}", updates.len());
};
assert_eq!(
update.action(),
&aws_sdk_lambda::types::OperationAction::Succeed
);
let payload = update.payload().unwrap_or_default();
assert_eq!(payload.len(), n);
assert!(
is_sized_result(payload, n),
"the payload is the serialized result"
);
let id = update.id();
assert_eq!(id.len(), 64, "{id}");
assert!(
id.chars()
.all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)),
"{id}"
);
let [end] = ends.as_slice() else {
unreachable!("one end, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Succeeded);
let recorded = end.execution_result().unwrap_or_default();
assert_eq!(recorded.len(), n);
assert!(
is_sized_result(recorded, n),
"the end info holds the full result"
);
}
#[tokio::test]
async fn oversized_result_write_failures() {
let n = LAMBDA_RESPONSE_SIZE_LIMIT + 1;
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(client::TestResponse::NonRetryableError(
"rejected".to_owned(),
));
let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
let response = response.expect("an envelope");
assert_eq!(json_str(&response, "/Status"), Some("FAILED"), "{response}");
assert_eq!(
json_str(&response, "/Error/ErrorType"),
Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
let message = json_str(&response, "/Error/ErrorMessage").unwrap_or_default();
assert!(
message.starts_with("[Unrecoverable Execution] Checkpoint failed: "),
"{message}"
);
let [end] = ends.as_slice() else {
unreachable!("one end, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Failed);
assert_eq!(end.execution_result(), None);
assert_eq!(
end.execution_error().and_then(WireError::error_type),
Some(error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
assert_eq!(
end.execution_error().and_then(WireError::error_message),
Some(message)
);
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(client::TestResponse::RetryableError(
"throttled".to_owned(),
));
let (response, ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
let err = response.expect_err("the invocation fails");
assert_eq!(
err.to_string(),
"[Unrecoverable Invocation] Checkpoint failed: throttled"
);
let [end] = ends.as_slice() else {
unreachable!("one end, got {}", ends.len());
};
assert_eq!(end.status(), InvocationStatus::Retrying);
assert_eq!(end.execution_result(), None);
assert_eq!(
end.execution_error().and_then(WireError::error_type),
Some(error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE)
);
assert_eq!(
end.execution_error().and_then(WireError::error_message),
Some("[Unrecoverable Invocation] Checkpoint failed: throttled")
);
}
#[tokio::test]
async fn oversized_result_after_withdrawal_is_pending() {
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
std::sync::Arc::new(engine::CheckpointLog::empty()),
std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>,
"t0".to_owned(),
Duration::ZERO,
0,
None,
);
ctx.coordinator()
.withdraw(ctx.execution_arn(), "request", || {});
let (response, end) = checkpoint_oversized_result(&ctx, "\"a\"".to_owned()).await;
assert_eq!(
response.expect("an envelope").0,
serde_json::json!({ "Status": "PENDING" })
);
assert_eq!(end.status, InvocationStatus::Pending);
assert_eq!(client.recorded_updates().len(), 0);
}
#[tokio::test]
async fn oversized_error_is_not_checkpointed() {
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
let options = Options::builder()
.execution_client(std::sync::Arc::clone(&client) as std::sync::Arc<dyn ExecutionClient>)
.build()
.expect("valid options");
let service = wrap(
|_event: serde_json::Value, _ctx: DurableContext| async {
Err::<String, BoxError>("e".repeat(7 * 1024 * 1024).into())
},
options,
);
let response = service(lambda_runtime::LambdaEvent::new(
InvocationPayload(oversized_envelope()),
lambda_runtime::Context::default(),
))
.await
.expect("an envelope")
.0;
assert_eq!(json_str(&response, "/Status"), Some("FAILED"));
assert_eq!(execution_updates(&client).len(), 0);
}
#[tokio::test]
async fn result_at_limit_stays_inline() {
let n = LAMBDA_RESPONSE_SIZE_LIMIT;
assert_eq!(n, 6_291_406);
let client = std::sync::Arc::new(client::InMemoryExecutionClient::new(Vec::new()));
let (response, _ends) = run_sized_result(n, std::sync::Arc::clone(&client)).await;
let response = response.expect("an envelope");
assert_eq!(json_str(&response, "/Status"), Some("SUCCEEDED"));
let result = json_str(&response, "/Result").unwrap_or_default();
assert_eq!(result.len(), n);
assert!(is_sized_result(result, n), "the result is inline");
assert_eq!(client.recorded_updates().len(), 0);
}
}