use std::future::Future;
use std::time::Duration;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
use crate::context::{DurableContext, StepContext, SubmittedWrite};
use crate::engine::{CheckpointStatus, OperationId};
use crate::error::{
OperationError, OperationErrorKind, STEP_INTERRUPTED_MESSAGE, StepError, StepErrorKind,
};
use crate::plugin::UserBody;
use crate::serdes::SerdesContext;
use crate::start::StartStep;
use crate::tracing_layer;
use crate::{BoxError, OperationSubType, RetryDecision, RetryStrategy, Serdes};
pub(crate) fn default_retry_strategy() -> RetryStrategy {
RetryStrategyConfig::default_preset().into_retry_strategy()
}
impl RetryStrategyConfig {
pub(crate) fn decide(&self, err: &StepError, attempt: u32) -> RetryDecision {
if attempt >= self.max_attempts() {
return RetryDecision::Stop;
}
if let Some(predicate) = self.retry_predicate()
&& !predicate(err)
{
return RetryDecision::Stop;
}
let initial = self.initial_delay().as_secs_f64();
let base = match self.backoff() {
Backoff::Exponential { rate } => {
let exponent = (i32::try_from(attempt).unwrap_or(1)) - 1;
initial * rate.powi(exponent)
}
Backoff::Linear { increment } => {
initial + increment.as_secs_f64() * f64::from(attempt.saturating_sub(1))
}
}
.min(self.max_delay().as_secs_f64());
let jittered = match self.jitter() {
JitterStrategy::None => base,
JitterStrategy::Half => base / 2.0 + rand_full_jitter(base / 2.0),
JitterStrategy::Full => rand_full_jitter(base),
};
RetryDecision::retry(quantize_delay(jittered))
}
pub(crate) fn into_retry_strategy(self) -> RetryStrategy {
Box::new(move |err: &StepError, attempt: u32| self.decide(err, attempt))
}
}
pub(crate) fn rand_full_jitter(max_secs: f64) -> f64 {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::time::SystemTime;
static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let mut hasher = DefaultHasher::new();
SystemTime::now().hash(&mut hasher);
std::thread::current().id().hash(&mut hasher);
COUNTER
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
.hash(&mut hasher);
let bits = hasher.finish();
#[expect(clippy::cast_precision_loss)] let fraction = (bits as f64) / (u64::MAX as f64);
fraction * max_secs
}
pub(crate) fn quantize_delay(secs: f64) -> Duration {
Duration::try_from_secs_f64(secs.round().max(1.0)).unwrap_or(Duration::from_secs(1))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum StepSemantics {
#[default]
AtLeastOncePerRetry,
AtMostOncePerRetry,
}
pub(crate) const STEP_SUB_TYPE: &str = "Step";
pub(crate) struct StepExecution<O, F, S> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) retry_strategy: Option<RetryStrategy>,
pub(crate) serdes: S,
pub(crate) semantics: StepSemantics,
pub(crate) sub_type: OperationSubType,
pub(crate) plugin_name: Option<String>,
pub(crate) closure: F,
pub(crate) _marker: std::marker::PhantomData<fn() -> O>,
}
impl<O, F, Fut, S> StepExecution<O, F, S>
where
O: Send + 'static,
F: FnOnce(StepContext) -> Fut + Clone + Send + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
S: Serdes<O>,
{
pub(crate) async fn execute(self) -> Result<O, OperationError> {
let (core, closure) = self.split();
Self::run(core, closure, None).await
}
pub(crate) fn execute_started(
self,
) -> impl Future<Output = Result<O, OperationError>> + Send + 'static {
let (core, closure) = self.split();
let first = core.start();
Self::run(core, closure, Some(first))
}
fn split(self) -> (StepCore<O, S>, F) {
let Self {
ctx,
op_id,
name,
retry_strategy,
serdes,
semantics,
sub_type,
plugin_name,
closure,
_marker,
} = self;
let core = StepCore {
ctx,
op_id,
name,
retry_strategy,
serdes,
semantics,
sub_type,
plugin_name,
_marker: std::marker::PhantomData,
};
(core, closure)
}
async fn run(
core: StepCore<O, S>,
mut closure: F,
mut first: Option<Result<StartStep, OperationError>>,
) -> Result<O, OperationError> {
let mut resumed = false;
loop {
match core.before(resumed, first.take()).await? {
StepPrelude::Done(result) => return result,
StepPrelude::Wait => {}
StepPrelude::Run {
attempt,
span,
start,
info,
} => {
let body = UserBody::Step(core.op_id.positional().to_owned());
let step_ctx = StepContext::new(attempt);
let closure = &mut closure;
let ran = core
.ctx
.hold_activity(crate::plugin::run_user_function(
core.ctx.shared_plugins(),
info,
body,
span,
move || {
let attempt_closure = (*closure).clone();
attempt_closure(step_ctx)
},
|error: &BoxError| {
crate::error::wire_error_for(&**error, STEP_FALLBACK_ERROR_TYPE)
},
))
.await;
core.await_start(start).await?;
match core.settle(attempt, ran).await? {
StepSettled::Done(value) => return Ok(value),
StepSettled::Retry => {}
}
}
}
core.ctx
.wait_for_record(core.op_id.wire(), |status| {
*status == CheckpointStatus::Ready || status.is_terminal()
})
.await;
resumed = true;
}
}
}
struct StepCore<O, S> {
ctx: DurableContext,
op_id: OperationId,
name: Option<String>,
retry_strategy: Option<RetryStrategy>,
serdes: S,
semantics: StepSemantics,
sub_type: OperationSubType,
plugin_name: Option<String>,
_marker: std::marker::PhantomData<fn() -> O>,
}
enum StepPrelude<O> {
Done(Result<O, OperationError>),
Wait,
Run {
attempt: u32,
span: tracing::Span,
start: Option<SubmittedWrite>,
info: std::sync::Arc<crate::plugin::OperationInfo>,
},
}
enum StepSettled<O> {
Done(O),
Retry,
}
impl<O, S> StepCore<O, S>
where
O: Send + 'static,
S: Serdes<O>,
{
fn emit_replayed(&self, resumed: bool, attempt: u32) {
if !resumed {
self.ctx.emit_operation_replayed(
self.op_id.wire(),
self.name.as_deref(),
"Step",
Some(self.sub_type.as_str()),
attempt,
);
}
}
fn start_update(&self) -> OperationUpdate {
build_start_update(
self.op_id.wire(),
self.name.as_deref(),
self.sub_type.as_str(),
self.ctx.parent_wire_id(),
)
}
async fn before(
&self,
resumed: bool,
first: Option<Result<StartStep, OperationError>>,
) -> Result<StepPrelude<O>, OperationError> {
let started = match first {
Some(started) => started,
None => self.start(),
}?;
let wire_id = self.op_id.wire();
self.after_start(resumed, started, wire_id).await
}
fn start(&self) -> Result<StartStep, OperationError> {
crate::start::start_step(
&self.ctx,
&crate::start::StartSpec {
op_id: &self.op_id,
operation_type: "Step",
sub_type: self.sub_type.as_str(),
name: self.name.as_deref(),
order: match self.semantics {
StepSemantics::AtLeastOncePerRetry => crate::start::StartOrder::QueueThenHook,
StepSemantics::AtMostOncePerRetry => crate::start::StartOrder::AwaitResponse,
},
},
|view| self.cycle(view),
|| self.start_update(),
|plugins, is_replay| self.operation_started(plugins, is_replay),
)
}
async fn after_start(
&self,
resumed: bool,
started: StartStep,
wire_id: &str,
) -> Result<StepPrelude<O>, OperationError> {
let start = match started {
StartStep::Refused => return std::future::pending().await,
StartStep::Queued(write) => Some(write),
StartStep::Recorded(view)
if view.status == CheckpointStatus::Started
&& self.semantics == StepSemantics::AtLeastOncePerRetry =>
{
None
}
StartStep::Recorded(view) => return self.recorded(resumed, &view).await,
};
let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
let start = match start {
None => None,
Some(start) if self.semantics == StepSemantics::AtLeastOncePerRetry => Some(start),
Some(start) => {
if let Err(err) = start.await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = self.ctx.plugins() {
self.operation_started(plugins, attempt > 1);
}
None
}
};
let is_replay = false; let span = tracing_layer::operation_span(
self.ctx.execution_arn(),
&self.ctx.lambda_context().request_id,
self.ctx.lambda_context().tenant_id.as_deref(),
wire_id,
self.name.as_deref(),
attempt,
is_replay,
);
let info = self.attempt_started(attempt);
Ok(StepPrelude::Run {
attempt,
span,
start,
info,
})
}
async fn recorded(
&self,
resumed: bool,
view: &crate::engine::CheckpointStatusView,
) -> Result<StepPrelude<O>, OperationError> {
let positional_id = self.op_id.positional();
let wire_id = self.op_id.wire();
match view.status {
CheckpointStatus::Succeeded => {
let serdes_ctx = SerdesContext::new(wire_id, self.ctx.execution_arn());
let payload = self.ctx.checkpoint_result_payload(positional_id);
let value = replay_success(
&self.ctx,
self.name.as_deref(),
&self.serdes,
payload,
serdes_ctx,
)
.await?;
self.emit_replayed(resumed, view.attempt);
Ok(StepPrelude::Done(Ok(value)))
}
CheckpointStatus::Failed => {
self.emit_replayed(resumed, view.attempt);
let wire = self
.ctx
.checkpoint_wire_error(positional_id)
.unwrap_or_default();
Ok(StepPrelude::Done(Err(replay_failure(wire, wire_id))))
}
CheckpointStatus::Started => {
let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
handle_failure(
&self.ctx,
wire_id,
self.name.as_deref(),
self.sub_type.as_str(),
self.plugin_name.as_deref(),
self.retry_strategy.as_ref(),
interrupted_failure(),
attempt,
)
.await?;
Ok(StepPrelude::Wait)
}
CheckpointStatus::Unknown(ref raw) => {
Err(self.ctx.unrecognized_status_error(wire_id, raw))
}
CheckpointStatus::Pending
| CheckpointStatus::Ready
| CheckpointStatus::Cancelled
| CheckpointStatus::TimedOut
| CheckpointStatus::Stopped => Ok(StepPrelude::Wait),
}
}
fn cycle(&self, view: &crate::engine::CheckpointStatusView) -> crate::start::Cycle {
step_cycle(self.semantics, view)
}
fn operation_started(&self, plugins: &crate::plugin::InvocationPlugins, is_replay: bool) {
let info = step_info(
&self.ctx,
self.op_id.wire(),
self.sub_type.as_str(),
self.name.as_deref(),
self.plugin_name.as_deref(),
)
.with_status(crate::OperationStatus::Started)
.with_replay(is_replay);
crate::plugin::operation_started(plugins, &info);
}
fn attempt_started(&self, attempt: u32) -> std::sync::Arc<crate::plugin::OperationInfo> {
let info = step_info(
&self.ctx,
self.op_id.wire(),
self.sub_type.as_str(),
self.name.as_deref(),
self.plugin_name.as_deref(),
);
match self.ctx.plugins() {
Some(plugins) => crate::plugin::attempt_start_only(plugins, info, attempt),
None => std::sync::Arc::new(info),
}
}
async fn await_start(&self, start: Option<SubmittedWrite>) -> Result<(), OperationError> {
if let Some(start) = start
&& let Err(err) = start.await
{
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
Ok(())
}
async fn settle(
&self,
attempt: u32,
ran: crate::plugin::Ran<O>,
) -> Result<StepSettled<O>, OperationError> {
let wire_id = self.op_id.wire();
match ran.into_parts() {
Ok(value) => handle_success(
&self.ctx,
wire_id,
self.name.as_deref(),
self.sub_type.as_str(),
self.plugin_name.as_deref(),
&self.serdes,
value,
attempt,
)
.await
.map(StepSettled::Done),
Err((err, wire)) => {
handle_failure(
&self.ctx,
wire_id,
self.name.as_deref(),
self.sub_type.as_str(),
self.plugin_name.as_deref(),
self.retry_strategy.as_ref(),
(StepError::execution_failed(err), wire),
attempt,
)
.await?;
Ok(StepSettled::Retry)
}
}
}
}
pub(crate) fn step_cycle(
semantics: StepSemantics,
view: &crate::engine::CheckpointStatusView,
) -> crate::start::Cycle {
use crate::start::Cycle;
match view.status {
CheckpointStatus::Ready
| CheckpointStatus::Cancelled
| CheckpointStatus::TimedOut
| CheckpointStatus::Stopped => Cycle::Start {
is_replay: view.attempt > 0,
},
CheckpointStatus::Started if semantics == StepSemantics::AtLeastOncePerRetry => {
Cycle::Resume
}
_ => Cycle::Record,
}
}
fn step_info(
ctx: &DurableContext,
wire_id: &str,
sub_type: &str,
name: Option<&str>,
plugin_name: Option<&str>,
) -> crate::plugin::OperationInfo {
let info = crate::plugin::OperationInfo::for_operation(
ctx,
wire_id,
crate::OperationKind::Step,
sub_type,
name,
);
match plugin_name {
Some(plugin_name) => info.with_name(Some(plugin_name.to_owned())),
None => info,
}
}
#[expect(clippy::too_many_arguments)] async fn handle_success<O, S: Serdes<O>>(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
sub_type: &str,
plugin_name: Option<&str>,
serdes: &S,
value: O,
attempt: u32,
) -> Result<O, OperationError> {
let serdes_ctx = SerdesContext::new(wire_id, ctx.execution_arn());
let serialized = match serialize_value(ctx, name, serdes, value, serdes_ctx.clone()).await {
Ok(serialized) => serialized,
Err(op_err) => {
let wire = crate::error::serialization_failure_wire(&op_err);
let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(client_err).await;
}
return Err(op_err
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire));
}
};
let update = build_succeed_update(wire_id, name, sub_type, ctx.parent_wire_id(), &serialized);
if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = ctx.plugins() {
crate::plugin::attempt_ended(
plugins,
step_info(ctx, wire_id, sub_type, name, plugin_name)
.with_status(crate::OperationStatus::Succeeded)
.with_result(Some(serialized.clone())),
crate::plugin::AttemptEnd {
attempt,
outcome: crate::plugin::AttemptOutcome::Succeeded,
error: None,
ends_operation: true,
},
);
}
deserialize_result(ctx, name, serdes, serialized, serdes_ctx).await
}
#[expect(clippy::too_many_arguments)] async fn handle_failure(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
sub_type: &str,
plugin_name: Option<&str>,
retry_strategy: Option<&RetryStrategy>,
(step_err, wire): (StepError, crate::error::WireError),
attempt: u32,
) -> Result<(), OperationError> {
let plugins = ctx
.plugins()
.filter(|_| !matches!(step_err.kind(), StepErrorKind::Interrupted));
let attempt_ended = |status: crate::OperationStatus, ends_operation: bool| {
if let Some(plugins) = plugins {
crate::plugin::attempt_ended(
plugins,
step_info(ctx, wire_id, sub_type, name, plugin_name)
.with_status(status)
.with_error(Some(wire.clone())),
crate::plugin::AttemptEnd {
attempt,
outcome: crate::plugin::AttemptOutcome::Failed,
error: Some(wire.clone()),
ends_operation,
},
);
}
};
let decision = if let Some(strategy) = retry_strategy {
strategy(&step_err, attempt)
} else {
default_retry_strategy()(&step_err, attempt)
};
match decision {
RetryDecision::Retry { delay } => {
let whole_secs = delay
.as_secs()
.saturating_add(u64::from(delay.subsec_nanos() > 0));
let delay_secs = i32::try_from(whole_secs.max(1)).unwrap_or(i32::MAX);
let update = build_retry_update(
wire_id,
name,
sub_type,
ctx.parent_wire_id(),
&wire,
delay_secs,
);
if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(client_err).await;
}
attempt_ended(crate::OperationStatus::Pending, false);
Ok(())
}
RetryDecision::Stop => {
let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(client_err).await;
}
attempt_ended(crate::OperationStatus::Failed, true);
Err(
OperationError::from_kind(OperationErrorKind::Step(step_err))
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire),
)
}
}
}
fn interrupted_failure() -> (StepError, crate::error::WireError) {
let message: BoxError = STEP_INTERRUPTED_MESSAGE.into();
let wire = crate::error::wire_error_for(&*message, STEP_INTERRUPTED_ERROR_TYPE);
(StepError::interrupted(), wire)
}
fn build_start_update(
wire_id: &str,
name: Option<&str>,
sub_type: &str,
parent_wire_id: Option<&str>,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(sub_type)
.action(OperationAction::Start);
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_succeed_update(
wire_id: &str,
name: Option<&str>,
sub_type: &str,
parent_wire_id: Option<&str>,
payload: &str,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(sub_type)
.action(OperationAction::Succeed)
.payload(payload);
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_retry_update(
wire_id: &str,
name: Option<&str>,
sub_type: &str,
parent_wire_id: Option<&str>,
error: &crate::error::WireError,
delay_secs: i32,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(sub_type)
.action(OperationAction::Retry)
.error(error.to_error_object())
.step_options(
aws_sdk_lambda::types::StepOptions::builder()
.next_attempt_delay_seconds(delay_secs)
.build(),
);
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_fail_update(
wire_id: &str,
name: Option<&str>,
sub_type: &str,
parent_wire_id: Option<&str>,
error: &crate::error::WireError,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(sub_type)
.action(OperationAction::Fail)
.error(error.to_error_object());
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
async fn serialize_value<O, S: Serdes<O>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &S,
value: O,
serdes_ctx: SerdesContext,
) -> Result<String, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx.hold_activity(serdes.serialize(value, serdes_ctx)).await {
Ok(serialized) => Ok(serialized),
Err(err) => Err(step_serialization_error(
ctx.route_serdes_error("serialize", &operation_id, name, err)
.await,
)),
}
}
async fn deserialize_result<O, S: Serdes<O>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &S,
serialized: String,
serdes_ctx: SerdesContext,
) -> Result<O, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx
.hold_activity(serdes.deserialize(serialized, serdes_ctx))
.await
{
Ok(value) => Ok(value),
Err(err) => Err(step_serialization_error(
ctx.route_serdes_error("deserialize", &operation_id, name, err)
.await,
)),
}
}
fn step_serialization_error(e: BoxError) -> OperationError {
OperationError::from_kind(OperationErrorKind::Step(StepError::new(
StepErrorKind::SerializationFailed,
Some(e),
)))
}
async fn replay_success<O, S: Serdes<O>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &S,
result: Option<String>,
serdes_ctx: SerdesContext,
) -> Result<O, OperationError> {
let payload = result.unwrap_or_else(|| "null".to_owned());
deserialize_result(ctx, name, serdes, payload, serdes_ctx).await
}
fn replay_failure(wire: crate::error::WireError, wire_id: &str) -> OperationError {
let kind = match wire.error_type() {
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE) => StepErrorKind::SerializationFailed,
Some(STEP_INTERRUPTED_ERROR_TYPE) => StepErrorKind::Interrupted,
_ => StepErrorKind::ExecutionFailed,
};
OperationError::from_kind(OperationErrorKind::Step(StepError::new(
kind,
Some(crate::error::ReplayedFailure::source_from(wire.clone())),
)))
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire)
}
const STEP_FALLBACK_ERROR_TYPE: &str = "Error";
const STEP_INTERRUPTED_ERROR_TYPE: &str = "StepInterruptedError";
#[cfg(test)]
#[expect(clippy::panic)]
#[expect(clippy::disallowed_methods)]
mod tests {
use super::*;
use crate::context::DurableContext;
use crate::engine::CheckpointLog;
use crate::plugin::recorder::UfHooks;
use std::sync::Arc;
#[tokio::test]
async fn replay_success_deserializes_json() {
let payload = "42".to_owned();
let ctx = SerdesContext::new("op-1", "arn:test");
let (dctx, _client) = context_over(Vec::new());
let result: Result<i32, OperationError> =
replay_success(&dctx, None, &crate::serdes::JsonSerdes, Some(payload), ctx).await;
assert_eq!(result.unwrap(), 42);
}
#[tokio::test]
async fn replay_success_null_returns_unit() {
let ctx = SerdesContext::new("op-1", "arn:test");
let (dctx, _client) = context_over(Vec::new());
let result: Result<(), OperationError> =
replay_success(&dctx, None, &crate::serdes::JsonSerdes, None, ctx).await;
assert!(result.is_ok());
}
#[test]
fn replay_failure_keeps_kind_and_wire_fields_apart() {
let wire = crate::error::WireError::new(Some("TypeError"), Some("bad input"));
let err = replay_failure(wire, "wire-1");
let OperationErrorKind::Step(step_err) = err.kind() else {
unreachable!("replay builds a step error");
};
let StepErrorKind::ExecutionFailed = step_err.kind() else {
unreachable!("ordinary failures replay as ExecutionFailed");
};
assert_eq!(err.to_string(), "operation error: step");
let source = std::error::Error::source(step_err).expect("synthetic source");
let replayed = source
.downcast_ref::<crate::error::ReplayedFailure>()
.expect("replay source is a ReplayedFailure");
assert_eq!(replayed.error_type(), Some("TypeError"));
assert_eq!(replayed.error_message(), Some("bad input"));
assert_eq!(err.operation_id(), Some("wire-1"));
assert_eq!(err.status(), Some("FAILED"));
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some("TypeError")
);
let chain = format!("{err:#}");
assert!(chain.contains("bad input"), "got: {chain}");
assert!(
!chain.contains("TypeError"),
"type must not fold into text: {chain}"
);
}
#[test]
fn replay_failure_empty_record_displays_unknown() {
let err = replay_failure(crate::error::WireError::default(), "wire-1");
let chain = format!("{err:#}");
assert!(chain.contains("unknown error"), "got: {chain}");
}
#[test]
fn replay_failure_serialization_discriminator_keeps_its_kind() {
let wire = crate::error::WireError::new(
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE),
Some("bad payload"),
);
let err = replay_failure(wire, "wire-1");
let OperationErrorKind::Step(step_err) = err.kind() else {
unreachable!("replay builds a step error");
};
assert!(matches!(
step_err.kind(),
StepErrorKind::SerializationFailed
));
}
#[tokio::test]
async fn serialize_deserialize_round_trip() {
let ctx = SerdesContext::new("op-1", "arn:test");
let (dctx, _client) = context_over(Vec::new());
let serialized = serialize_value(
&dctx,
None,
&crate::serdes::JsonSerdes,
"hello".to_owned(),
ctx.clone(),
)
.await
.unwrap();
let deserialized: String =
deserialize_result(&dctx, None, &crate::serdes::JsonSerdes, serialized, ctx)
.await
.unwrap();
assert_eq!(deserialized, "hello");
}
#[tokio::test]
async fn serialize_with_custom_serdes_uppercases() {
struct Upper;
impl Serdes<String> for Upper {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(serde_json::to_string(&value)?.to_uppercase())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(serde_json::from_str(&wire)?)
}
}
let ctx = SerdesContext::new("op-1", "arn:test");
let (dctx, _client) = context_over(Vec::new());
let result = serialize_value(&dctx, None, &Upper, "hello".to_owned(), ctx)
.await
.unwrap();
assert_eq!(result, "\"HELLO\"");
}
#[test]
fn default_retry_stops_at_max_attempts() {
let strategy = default_retry_strategy();
let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
let decision = strategy(&err, 6);
assert_eq!(decision, RetryDecision::Stop);
}
#[test]
fn default_retry_retries_below_max() {
let strategy = default_retry_strategy();
let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
let decision = strategy(&err, 1);
match decision {
RetryDecision::Retry { delay } => {
assert!(delay.as_secs() >= 1);
assert!(delay.as_secs() <= 5);
}
RetryDecision::Stop => panic!("expected retry for attempt 1"),
}
}
#[test]
fn default_retry_delay_grows_with_attempts() {
let strategy = default_retry_strategy();
let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
let decision = strategy(&err, 5);
match decision {
RetryDecision::Retry { delay } => {
assert!(delay.as_secs() <= 60);
assert!(delay.as_secs() >= 1);
}
RetryDecision::Stop => panic!("expected retry for attempt 5"),
}
}
#[tokio::test]
async fn step_live_success_checkpoints_and_returns() {
use crate::client::InMemoryExecutionClient;
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let closure = |_: StepContext| async { Ok::<i32, BoxError>(42) };
let exec = StepExecution {
ctx,
op_id,
name: Some("test-step".to_owned()),
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), 42);
}
#[tokio::test]
async fn per_operation_serdes_serializes_step_result() {
use crate::client::InMemoryExecutionClient;
struct Upper;
impl Serdes<String> for Upper {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(serde_json::to_string(&value)?.to_uppercase())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(serde_json::from_str(&wire.to_lowercase())?)
}
}
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client.clone(),
"token0".to_owned(),
Duration::ZERO,
0,
None,
);
let op_id = ctx.mint_id();
let closure = |_: StepContext| async { Ok::<String, BoxError>("hello".to_owned()) };
let exec = StepExecution {
ctx,
op_id,
name: Some("greet".to_owned()),
retry_strategy: None,
serdes: Upper,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), "hello");
let updates = client.recorded_updates();
let succeed = updates
.iter()
.find(|u| matches!(u.action(), OperationAction::Succeed))
.expect("a Succeed update must be recorded");
assert_eq!(
succeed.payload(),
Some("\"HELLO\""),
"the per-operation serdes must serialize the step result"
);
}
#[tokio::test]
async fn step_replay_returns_frozen_result_without_re_executing() {
use crate::engine::{CheckpointLog, CheckpointRecord};
use std::sync::atomic::{AtomicBool, Ordering};
let executed = Arc::new(AtomicBool::new(false));
let executed_clone = Arc::clone(&executed);
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Succeeded,
result: Some("99".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let ctx = DurableContext::new_root(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
);
let op_id = ctx.mint_id();
let closure = move |_: StepContext| {
executed_clone.store(true, Ordering::SeqCst);
async { Ok::<i32, BoxError>(0) }
};
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), 99);
assert!(
!executed.load(Ordering::SeqCst),
"closure should NOT execute during replay"
);
}
#[tokio::test]
async fn step_replay_failure_returns_error_without_re_executing() {
use crate::engine::{CheckpointLog, CheckpointRecord};
use std::sync::atomic::{AtomicBool, Ordering};
let executed = Arc::new(AtomicBool::new(false));
let executed_clone = Arc::clone(&executed);
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Failed,
result: None,
error_type: Some("CustomError".to_owned()),
error_message: Some("it broke".to_owned()),
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let ctx = DurableContext::new_root(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
);
let op_id = ctx.mint_id();
let closure = move |_: StepContext| {
executed_clone.store(true, Ordering::SeqCst);
async { Ok::<String, BoxError>("should not run".to_owned()) }
};
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert!(result.is_err());
let err = result.unwrap_err();
let err_msg = format!("{err:#}");
assert!(err_msg.contains("it broke"), "got: {err_msg}");
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some("CustomError")
);
assert!(
!executed.load(Ordering::SeqCst),
"closure should NOT execute during replay"
);
}
#[tokio::test]
async fn step_retry_then_stop_on_exhaustion() {
use crate::client::InMemoryExecutionClient;
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let no_retry: RetryStrategy =
Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
let closure = |_: StepContext| async { Err::<i32, BoxError>("always fails".into()) };
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: Some(no_retry),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert!(result.is_err());
let err_msg = format!("{:#}", result.unwrap_err());
assert!(
err_msg.contains("retries exhausted") || err_msg.contains("always fails"),
"got: {err_msg}"
);
}
fn context_with_plugin(
hooks: &crate::plugin::test_support::OpHooks,
window: Duration,
) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
window,
0,
Some(hooks.plugins()),
);
(ctx, client)
}
#[tokio::test]
async fn hooks_run_outside_the_token_lock() {
let lock: Arc<std::sync::OnceLock<Arc<tokio::sync::Mutex<String>>>> = Arc::default();
let probe_lock = Arc::clone(&lock);
let hooks = crate::plugin::test_support::OpHooks::probing(move || {
let free = probe_lock
.get()
.is_some_and(|mutex| mutex.try_lock().is_ok());
free.to_string()
});
let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
lock.set(ctx.checkpoint_token_mutex()).expect("set once");
let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
assert_eq!(value, 1);
let probes: Vec<String> = hooks.records().into_iter().map(|r| r.probe).collect();
assert_eq!(probes, ["true"; 4]);
}
#[tokio::test]
async fn hooks_run_on_the_operation_task() {
let hooks = crate::plugin::test_support::OpHooks::probing(|| {
format!("{:?}", tokio::task::try_id())
});
let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
let body_task = Arc::new(std::sync::Mutex::new(None));
let seen = Arc::clone(&body_task);
let value = ctx
.step(move |_| async move {
*seen.lock().unwrap() = tokio::task::try_id();
Ok(1_u32)
})
.name("a")
.spawn()
.await
.unwrap();
assert_eq!(value, 1);
let body = body_task.lock().unwrap().expect("the body ran on a task");
let records = hooks.records();
let probe_of = |hook: &str| {
records
.iter()
.find(|r| r.hook == hook)
.map(|r| r.probe.clone())
.expect("the hook ran")
};
let expected = format!("{:?}", Some(body));
assert_eq!(probe_of("attempt_end"), expected);
assert_eq!(probe_of("end"), expected);
}
#[tokio::test]
async fn no_plugins_keeps_checkpoint_updates() {
let (ctx, client) = context_over(Vec::new());
assert!(ctx.plugins().is_none());
let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
assert_eq!(value, 1);
let updates = client.recorded_updates();
let wire_id = crate::engine::compute_wire_id_public("1");
let fields: Vec<_> = updates
.iter()
.map(|u| {
(
u.id().to_owned(),
u.r#type().clone(),
u.sub_type().map(str::to_owned),
u.name().map(str::to_owned),
u.action().clone(),
u.payload().map(str::to_owned),
u.parent_id().map(str::to_owned),
u.error().is_some(),
)
})
.collect();
let step = |action, payload: Option<&str>| {
(
wire_id.clone(),
OperationType::Step,
Some(STEP_SUB_TYPE.to_owned()),
Some("a".to_owned()),
action,
payload.map(str::to_owned),
None,
false,
)
};
assert_eq!(
fields,
[
step(OperationAction::Start, None),
step(OperationAction::Succeed, Some("1")),
]
);
}
#[tokio::test]
async fn failed_step_end_infos_carry_the_written_error() {
let hooks = crate::plugin::test_support::OpHooks::default();
let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
let err = ctx
.step(|_| async { Err::<u32, BoxError>("boom".into()) })
.name("a")
.retry_strategy(|_err, _attempt| RetryDecision::Stop)
.await
.unwrap_err();
assert!(err.wire().is_some());
let records = hooks.records();
let ends: Vec<_> = records
.iter()
.filter(|r| r.hook == "attempt_end" || r.hook == "end")
.collect();
assert_eq!(ends.len(), 2);
let attempt_end = ends.first().expect("an attempt end");
assert_eq!(attempt_end.error.as_deref(), Some("boom"));
assert_eq!(attempt_end.status, Some(crate::OperationStatus::Failed));
}
fn uf_context(
hooks: &UfHooks,
records: Vec<(String, crate::engine::CheckpointRecord)>,
) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
Duration::ZERO,
0,
Some(hooks.plugins()),
);
(ctx, client)
}
async fn drive<F>(ctx: &DurableContext, fut: F) -> crate::driver::InvocationOutcome
where
F: IntoFuture + Send + 'static,
F::IntoFuture: Send + 'static,
{
let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
crate::driver::test_support::outcome_of(ctx.clone(), ownership, signal, fut).await
}
fn update_of(
client: &crate::client::InMemoryExecutionClient,
name: &str,
action: &OperationAction,
) -> Option<OperationUpdate> {
client
.recorded_updates()
.into_iter()
.find(|u| u.name() == Some(name) && u.action() == action)
}
fn type_and_message(wire: &crate::error::WireError) -> (Option<String>, Option<String>) {
(
wire.error_type().map(str::to_owned),
wire.error_message().map(str::to_owned),
)
}
fn update_error(update: &OperationUpdate) -> (Option<String>, Option<String>) {
let error = update.error().expect("the update carries an error object");
(
error.error_type().map(str::to_owned),
error.error_message().map(str::to_owned),
)
}
const LIVE_ATTEMPT: [&str; 6] = [
"attempt_start",
"uf_start",
"wrap_enter",
"wrap_exit",
"uf_end",
"attempt_end",
];
#[tokio::test]
async fn interrupted_attempt_fires_no_user_function_hook() {
use std::sync::atomic::{AtomicU32, Ordering};
let runs = Arc::new(AtomicU32::new(0));
let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
let runs = Arc::clone(runs);
ctx.step(move |_| async move {
runs.fetch_add(1, Ordering::SeqCst);
Ok(42_u32)
})
.name("s")
.semantics(StepSemantics::AtMostOncePerRetry)
.retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(1)))
};
let hooks = UfHooks::default();
let (ctx, client) = uf_context(
&hooks,
vec![step_record(CheckpointStatus::Started, 0, None)],
);
let outcome = drive(&ctx, step(&ctx, &runs)).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_eq!(runs.load(Ordering::SeqCst), 0, "the body did not run");
let retry = recorded_error_for(&client, &OperationAction::Retry);
assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
let hooks = UfHooks::default();
let (ctx, _client) =
uf_context(&hooks, vec![step_record(CheckpointStatus::Ready, 1, None)]);
assert_eq!(step(&ctx, &runs).await.unwrap(), 42);
let mut expected = vec!["start"];
expected.extend(LIVE_ATTEMPT);
expected.push("end");
assert_eq!(hooks.hooks_of("s"), expected);
assert_eq!(hooks.find("s", "uf_start").info_attempt, Some(2));
assert_eq!(hooks.find("s", "uf_end").ok, Some(true));
}
#[tokio::test]
async fn retry_wait_fires_no_user_function_hook() {
use std::sync::atomic::{AtomicU32, Ordering};
let runs = Arc::new(AtomicU32::new(0));
let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
let runs = Arc::clone(runs);
ctx.step(move |_| async move {
runs.fetch_add(1, Ordering::SeqCst);
Err::<u32, BoxError>("transient".into())
})
.name("s")
.retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(10)))
};
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let outcome = drive(&ctx, step(&ctx, &runs)).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_eq!(runs.load(Ordering::SeqCst), 1);
let mut expected = vec!["start"];
expected.extend(LIVE_ATTEMPT);
assert_eq!(hooks.hooks_of("s"), expected);
assert_eq!(hooks.find("s", "uf_end").ok, Some(false));
assert!(update_of(&client, "s", &OperationAction::Retry).is_some());
let hooks = UfHooks::default();
let (wire_key, mut record) = step_record(CheckpointStatus::Pending, 1, None);
record.due_at = Some(std::time::SystemTime::now() + Duration::from_secs(10));
let (ctx, client) = uf_context(&hooks, vec![(wire_key, record)]);
let outcome = drive(&ctx, step(&ctx, &runs)).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_eq!(
runs.load(Ordering::SeqCst),
1,
"no body ran in invocation 2"
);
assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
assert!(
client.recorded_updates().is_empty(),
"the step stays PENDING: {:?}",
client.recorded_updates()
);
}
#[tokio::test]
async fn end_hook_before_outcome_record() {
let client_cell: Arc<std::sync::OnceLock<Arc<crate::client::InMemoryExecutionClient>>> =
Arc::default();
let probe_client = Arc::clone(&client_cell);
let hooks = UfHooks::probing(move || {
let actions: Vec<String> = probe_client
.get()
.map(|client| client.recorded_updates())
.unwrap_or_default()
.iter()
.map(|u| format!("{:?}", u.action()))
.collect();
actions.join(",")
});
let (ctx, client) = uf_context(&hooks, Vec::new());
client_cell.set(Arc::clone(&client)).expect("set once");
let body_client = Arc::clone(&client);
let value = ctx
.step(move |_| async move {
for _ in 0..1000 {
if update_of(&body_client, "s", &OperationAction::Start).is_some() {
return Ok(1_u32);
}
tokio::task::yield_now().await;
}
Err("the START was never written".into())
})
.name("s")
.await
.unwrap();
assert_eq!(value, 1);
assert_eq!(hooks.find("s", "uf_end").probe, "Start");
assert!(update_of(&client, "s", &OperationAction::Succeed).is_some());
}
#[tokio::test]
async fn failed_body_wire_derived_once() {
use crate::plugin::recorder::Counted;
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let dc = Counted::new();
let body_dc = dc.clone();
let err = ctx
.step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
.name("step")
.retry_strategy(|_err, _attempt| RetryDecision::Stop)
.await
.unwrap_err();
drop(err);
assert_eq!(dc.calls(), 1, "step");
let end = hooks.find("step", "uf_end").wire.expect("an Err outcome");
let attempt_end = hooks.find("step", "attempt_end").wire.expect("an error");
assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
assert_eq!(type_and_message(&end), update_error(&fail));
assert_eq!(end.error_message(), Some("counted"));
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let dc = Counted::new();
let body_dc = dc.clone();
let err = ctx
.run_in_child_context(
move |_child| async move { Err::<u32, BoxError>(body_dc.boxed()) },
)
.name("child")
.await
.unwrap_err();
drop(err);
assert_eq!(dc.calls(), 1, "child");
let end = hooks.find("child", "uf_end").wire.expect("an Err outcome");
let fail = update_of(&client, "child", &OperationAction::Fail).expect("a FAIL");
assert_eq!(type_and_message(&end), update_error(&fail));
let dc = Counted::new();
let probe_dc = dc.clone();
let hooks = UfHooks::probing(move || probe_dc.calls().to_string());
let (ctx, client) = uf_context(&hooks, Vec::new());
let body_dc = dc.clone();
let err = ctx
.wait_for_callback::<String>(move |_step_ctx, _callback_id| async move {
Err(body_dc.boxed())
})
.name("cb")
.submitter_retry(|_err, _attempt| RetryDecision::Stop)
.await
.unwrap_err();
drop(err);
assert_eq!(hooks.find("cb-submitter", "uf_end").probe, "1");
assert_eq!(hooks.find("cb-submitter", "end").probe, "1");
assert_eq!(dc.calls(), 1, "the submitter only");
assert_eq!(hooks.find("cb", "uf_end").probe, "1");
let outer = hooks.find("cb", "uf_end").wire.expect("an Err outcome");
assert_eq!(outer.error_message(), Some("counted"));
let end = hooks
.find("cb-submitter", "uf_end")
.wire
.expect("an Err outcome");
let attempt_end = hooks
.find("cb-submitter", "attempt_end")
.wire
.expect("an error");
assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
assert_eq!(end.error_message(), Some("counted"));
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.r#type() == &OperationType::Step && u.action() == &OperationAction::Fail)
.expect("the submitter's FAIL");
assert_eq!(type_and_message(&end), update_error(&fail));
map_item_wire_derived_once().await;
}
#[tokio::test]
async fn failed_body_records_a_propagated_record_unchanged() {
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let err = ctx
.step(|_| async {
let wire = crate::error::WireError::new(Some("InnerType"), Some("inner"))
.with_error_data(Some("inner-data"))
.with_stack_trace(vec!["inner-frame".to_owned()]);
Err::<u32, BoxError>(Box::new(crate::error::recorded_step_error(wire)))
})
.name("step")
.retry_strategy(|_err, _attempt| RetryDecision::Stop)
.await
.unwrap_err();
drop(err);
let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
let error = fail.error().expect("the FAIL carries an error object");
assert_eq!(error.error_type(), Some("InnerType"));
assert_eq!(error.error_message(), Some("inner"));
assert_eq!(error.error_data(), Some("inner-data"));
assert_eq!(error.stack_trace(), ["inner-frame"]);
}
async fn map_item_wire_derived_once() {
use crate::builders::map_parallel::CompletionConfig;
use crate::map_parallel::NestingMode;
use crate::plugin::recorder::Counted;
for mode in [NestingMode::Normal, NestingMode::Flat] {
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let dc = Counted::new();
let body_dc = dc.clone();
let batch = ctx
.map(vec![1_u32], move |_child, _item, _idx| {
let dc = body_dc.clone();
async move { Err::<u32, BoxError>(dc.boxed()) }
})
.name("m")
.nesting(mode)
.completion(CompletionConfig::with_tolerated_failure_count(1))
.await_batch()
.await
.unwrap();
assert_eq!(dc.calls(), 1, "{mode:?}");
let end = hooks
.find("map-item-0", "uf_end")
.wire
.expect("an Err outcome");
let item = batch.errors().next().expect("the item failed");
assert_eq!(
type_and_message(&end),
(
item.error_type.map(str::to_owned),
Some(item.message.to_owned())
),
"{mode:?}"
);
let fail = update_of(&client, "map-item-0", &OperationAction::Fail);
match mode {
NestingMode::Normal => {
assert_eq!(type_and_message(&end), update_error(&fail.expect("a FAIL")));
}
NestingMode::Flat => assert!(fail.is_none(), "a flat item writes no record"),
}
}
}
#[tokio::test]
async fn failing_step_display_calls_durable_op() {
use crate::plugin::recorder::Counted;
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let claim_ctx = ctx.clone();
let dc = Counted::calling(move || drop(claim_ctx.wait(Duration::from_secs(1))));
let body_dc = dc.clone();
let step = ctx
.step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
.name("s")
.retry_strategy(|_err, _attempt| RetryDecision::Stop);
assert!(
tokio::time::timeout(Duration::from_millis(50), step.into_future())
.await
.is_err(),
"the step's FAIL is refused, so the step never resolves"
);
assert_eq!(dc.calls(), 1);
assert!(
update_of(&client, "s", &OperationAction::Fail).is_none(),
"no terminal update is recorded"
);
assert!(update_of(&client, "s", &OperationAction::Succeed).is_none());
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert!(
fatal
.error_message
.contains("while the SDK records the outcome of operation \"1\""),
"{}",
fatal.error_message
);
}
struct FailingSerializer;
impl<T: Send + 'static> Serdes<T> for FailingSerializer {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(&self, _value: T, _context: SerdesContext) -> Result<String, BoxError> {
Err("cannot encode".into())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(&self, _wire: String, _context: SerdesContext) -> Result<T, BoxError> {
Err("cannot decode".into())
}
}
#[tokio::test]
async fn serialization_failure_keeps_its_own_wire() {
let hooks = UfHooks::default();
let (ctx, client) = uf_context(&hooks, Vec::new());
let err = ctx
.step(|_| async { Ok(1_u32) })
.name("s")
.serdes(FailingSerializer)
.await
.unwrap_err();
drop(err);
let end = hooks.find("s", "uf_end");
assert_eq!(end.ok, Some(true));
assert_eq!(end.wire, None);
let fail = update_of(&client, "s", &OperationAction::Fail).expect("a FAIL");
let (error_type, _) = update_error(&fail);
assert_eq!(
error_type.as_deref(),
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE)
);
}
fn step_record(
status: CheckpointStatus,
attempt: u32,
error: Option<&crate::error::WireError>,
) -> (String, crate::engine::CheckpointRecord) {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = crate::engine::CheckpointRecord {
id: wire_key.clone(),
status,
result: None,
error_type: error
.and_then(crate::error::WireError::error_type)
.map(str::to_owned),
error_message: error
.and_then(crate::error::WireError::error_message)
.map(str::to_owned),
error_data: error
.and_then(crate::error::WireError::error_data)
.map(str::to_owned),
stack_trace: None,
attempt,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
(wire_key, record)
}
fn context_over(
records: Vec<(String, crate::engine::CheckpointRecord)>,
) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
(ctx, client)
}
fn recorded_fail(client: &crate::client::InMemoryExecutionClient) -> crate::error::WireError {
let updates = client.recorded_updates();
let fail = updates
.iter()
.find(|u| matches!(u.action(), OperationAction::Fail))
.expect("a FAIL update must be recorded");
let error = fail
.error()
.expect("the FAIL update carries an error object");
assert!(
error.stack_trace().is_empty(),
"the SDK captures no stack trace: {error:?}"
);
crate::error::WireError::new(error.error_type(), error.error_message())
.with_error_data(error.error_data())
}
fn payment_step(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
let op_id = ctx.mint_id();
StepExecution {
ctx,
op_id,
name: Some("pay".to_owned()),
retry_strategy: Some(Box::new(|_err: &StepError, attempt: u32| {
if attempt >= 2 {
RetryDecision::Stop
} else {
RetryDecision::retry(Duration::from_secs(1))
}
})),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
plugin_name: None,
closure: |step_ctx: StepContext| async move {
if step_ctx.attempt() == 1 {
Err::<i32, BoxError>("transient".into())
} else {
Ok(7)
}
},
_marker: std::marker::PhantomData,
}
.execute()
}
#[tokio::test]
async fn custom_sub_type_on_every_step_update() {
let (ctx, first) = context_over(Vec::new());
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
payment_step(ctx),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let (id, mut record) = step_record(CheckpointStatus::Ready, 1, None);
record.sub_type = Some("Payment".to_owned());
let (ctx, second) = context_over(vec![(id, record)]);
assert_eq!(payment_step(ctx).await.expect("attempt 2 succeeds"), 7);
let updates: Vec<OperationUpdate> = first
.recorded_updates()
.into_iter()
.chain(second.recorded_updates())
.collect();
let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
assert_eq!(
actions,
[
OperationAction::Start,
OperationAction::Retry,
OperationAction::Start,
OperationAction::Succeed
]
);
for update in &updates {
assert_eq!(update.sub_type(), Some("Payment"), "{update:?}");
}
}
#[tokio::test]
async fn custom_sub_type_on_rejected_write_and_no_terminal_fail() {
use crate::client::TestResponse;
let (ctx, client) = context_over(Vec::new());
client.enqueue_checkpoint_response(TestResponse::Success(vec![]));
client.enqueue_checkpoint_response(TestResponse::NonRetryableError(
"payload rejected".to_owned(),
));
let op_id = ctx.mint_id();
let exec = StepExecution {
ctx: ctx.clone(),
op_id,
name: Some("pay".to_owned()),
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
plugin_name: None,
closure: |_: StepContext| async { Ok::<i32, BoxError>(7) },
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
)
.await;
assert!(
matches!(outcome, crate::driver::InvocationOutcome::Failed { .. }),
"a rejected SUCCEED fails the execution: {outcome:?}"
);
let updates = client.recorded_updates();
let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
assert_eq!(actions, [OperationAction::Start, OperationAction::Succeed]);
let rejected = updates.last().expect("the rejected SUCCEED is recorded");
assert_eq!(rejected.sub_type(), Some("Payment"));
}
async fn typed_failing_body(_: StepContext) -> Result<i32, BoxError> {
Err(Box::new(
crate::error::TypedError::with_type("PaymentError", "declined")
.with_error_data(r#"{"code":42}"#),
))
}
#[tokio::test]
async fn step_failure_records_typed_error_data() {
let (ctx, client) = context_over(Vec::new());
let op_id = ctx.mint_id();
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: Some(Box::new(|_err: &StepError, _attempt: u32| {
RetryDecision::Stop
})),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: typed_failing_body,
_marker: std::marker::PhantomData,
};
let live = exec.execute().await.expect_err("the step must fail");
let recorded = recorded_fail(&client);
assert_eq!(recorded.error_type(), Some("PaymentError"));
assert_eq!(recorded.error_message(), Some("declined"));
assert_eq!(recorded.error_data(), Some(r#"{"code":42}"#));
assert_eq!(
live.wire().and_then(crate::error::WireError::error_data),
Some(r#"{"code":42}"#)
);
assert_eq!(
live.wire().and_then(crate::error::WireError::error_message),
Some("declined")
);
let replayed_wire = crate::error::WireError::new(Some("PaymentError"), Some("declined"))
.with_error_data(Some(r#"{"code":42}"#));
let (ctx, _client) = context_over(vec![step_record(
CheckpointStatus::Failed,
0,
Some(&replayed_wire),
)]);
let op_id = ctx.mint_id();
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: typed_failing_body,
_marker: std::marker::PhantomData,
};
let err = exec
.execute()
.await
.expect_err("the replayed step must fail");
assert_eq!(
err.wire().and_then(crate::error::WireError::error_data),
Some(r#"{"code":42}"#)
);
assert_eq!(
err.wire().and_then(crate::error::WireError::error_message),
Some("declined")
);
}
#[tokio::test]
async fn step_stop_returns_execution_failed_live_and_replay() {
fn retry_once() -> RetryStrategy {
Box::new(|_err: &StepError, attempt: u32| {
if attempt >= 2 {
RetryDecision::Stop
} else {
RetryDecision::retry(Duration::from_secs(1))
}
})
}
fn execution(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
let op_id = ctx.mint_id();
StepExecution {
ctx,
op_id,
name: None,
retry_strategy: Some(retry_once()),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
_marker: std::marker::PhantomData,
}
.execute()
}
fn assert_execution_failed(err: &OperationError) {
let OperationErrorKind::Step(step_err) = err.kind() else {
unreachable!("a step builds a step error: {err:?}");
};
assert!(
matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
"unexpected kind: {:?}",
step_err.kind()
);
let rendered = format!("{err:#}");
assert!(rendered.ends_with("boom"), "got: {rendered}");
assert!(!rendered.contains("retries exhausted"), "got: {rendered}");
}
let (ctx, client) = context_over(Vec::new());
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
execution(ctx),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert!(
client
.recorded_updates()
.iter()
.any(|u| matches!(u.action(), OperationAction::Retry)),
"attempt 1 records a RETRY"
);
let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Ready, 1, None)]);
let live = execution(ctx).await.expect_err("the step must fail");
assert_execution_failed(&live);
assert_eq!(
format!("{live:#}"),
"operation error: step: execution failed: boom"
);
let recorded = recorded_fail(&client);
let (ctx, _client) = context_over(vec![step_record(
CheckpointStatus::Failed,
1,
Some(&recorded),
)]);
let replayed = execution(ctx)
.await
.expect_err("the replayed step must fail");
assert_execution_failed(&replayed);
}
#[tokio::test]
async fn step_live_failure_source_downcasts_to_concrete_user_type() {
use crate::client::InMemoryExecutionClient;
#[derive(Debug)]
struct PaymentDeclined {
code: u16,
}
impl std::fmt::Display for PaymentDeclined {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "payment declined (code {})", self.code)
}
}
impl std::error::Error for PaymentDeclined {}
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let no_retry: RetryStrategy =
Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
let closure = |_: StepContext| async {
Err::<i32, BoxError>(Box::new(PaymentDeclined { code: 402 }))
};
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: Some(no_retry),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("step must fail");
let mut source: Option<&(dyn std::error::Error + 'static)> =
std::error::Error::source(&err);
let mut found = None;
while let Some(e) = source {
if let Some(declined) = e.downcast_ref::<PaymentDeclined>() {
found = Some(declined);
break;
}
source = e.source();
}
let declined = found.expect("caller's concrete error type must be reachable via source()");
assert_eq!(declined.code, 402);
assert_eq!(err.status(), Some("FAILED"));
let wire = err.wire().expect("live failure carries its wire record");
assert_eq!(wire.error_type(), Some("Error"));
assert!(
wire.error_message()
.is_some_and(|m| m.contains("payment declined (code 402)")),
"wire message flattens the chain: {wire:?}"
);
assert!(
wire.stack_trace().is_empty(),
"the SDK captures no stack trace for a live failure"
);
}
#[tokio::test]
async fn step_retry_schedules_and_suspends() {
use crate::client::InMemoryExecutionClient;
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let always_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
RetryDecision::retry(Duration::from_secs(1))
});
let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
let exec = StepExecution {
ctx: ctx.clone(),
op_id,
name: None,
retry_strategy: Some(always_retry),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
}
#[tokio::test]
async fn step_retry_fractional_delay_rounds_up() {
use crate::client::InMemoryExecutionClient;
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client.clone(),
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let fractional_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
RetryDecision::retry(Duration::from_millis(1900))
});
let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
let exec = StepExecution {
ctx: ctx.clone(),
op_id,
name: None,
retry_strategy: Some(fractional_retry),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let updates = client.recorded_updates();
let retry = updates
.iter()
.find(|u| matches!(u.action(), OperationAction::Retry))
.expect("a Retry update must be recorded");
let delay = retry
.step_options()
.and_then(aws_sdk_lambda::types::StepOptions::next_attempt_delay_seconds)
.expect("Retry update must carry NextAttemptDelaySeconds");
assert_eq!(delay, 2, "1.9s must round UP to 2s, not truncate to 1s");
}
#[tokio::test]
async fn step_spawn_executes_on_blessed_task() {
use crate::client::InMemoryExecutionClient;
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let handle = ctx.step(|_| async { Ok(7i32) }).name("spawned").spawn();
let result = handle.await;
assert_eq!(result.unwrap(), 7);
}
#[tokio::test]
async fn step_ownership_rejection_from_foreign_task() {
use crate::client::InMemoryExecutionClient;
let result = tokio::spawn(async {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let ctx_clone = ctx.clone();
let handle = tokio::spawn(async move {
let op_id = ctx_clone.mint_id();
let closure = |_: StepContext| async { Ok::<i32, BoxError>(1) };
let exec = StepExecution {
ctx: ctx_clone,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::default(),
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
exec.execute().await
});
handle.await.unwrap()
})
.await;
let inner_result = result.unwrap();
assert!(inner_result.is_err());
let err_msg = format!("{:#}", inner_result.unwrap_err());
assert!(
err_msg.contains("task") || err_msg.contains("ownership"),
"expected ownership error, got: {err_msg}"
);
}
fn kind_recording_strategy(
decision: RetryDecision,
) -> (RetryStrategy, Arc<std::sync::Mutex<Vec<String>>>) {
let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
let seen_clone = Arc::clone(&seen);
let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
seen_clone
.lock()
.expect("kind log lock")
.push(err.kind().to_string());
decision.clone()
});
(strategy, seen)
}
fn recorded_error_for(
client: &crate::client::InMemoryExecutionClient,
action: &OperationAction,
) -> crate::error::WireError {
let updates = client.recorded_updates();
let update = updates
.iter()
.find(|u| u.action() == action)
.unwrap_or_else(|| panic!("a {action:?} update must be recorded: {updates:?}"));
let error = update.error().expect("the update carries an error object");
crate::error::WireError::new(error.error_type(), error.error_message())
}
fn at_most_once_over_started(
ctx: DurableContext,
retry_strategy: Option<RetryStrategy>,
executed: Arc<std::sync::atomic::AtomicBool>,
) -> StepExecution<
i32,
impl FnOnce(StepContext) -> std::future::Ready<Result<i32, BoxError>> + Clone,
crate::serdes::JsonSerdes,
> {
let op_id = ctx.mint_id();
StepExecution {
ctx,
op_id,
name: Some("at-most-once-step".to_owned()),
retry_strategy,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::AtMostOncePerRetry,
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: move |_: StepContext| {
executed.store(true, std::sync::atomic::Ordering::SeqCst);
std::future::ready(Ok::<i32, BoxError>(42))
},
_marker: std::marker::PhantomData,
}
}
#[tokio::test]
async fn step_at_most_once_replay_started_no_retry_fails_permanently() {
use std::sync::atomic::{AtomicBool, Ordering};
let executed = Arc::new(AtomicBool::new(false));
let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
let err = exec
.execute()
.await
.expect_err("expected failure for interrupted + no retry");
assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
let fail = recorded_error_for(&client, &OperationAction::Fail);
assert_eq!(fail.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
assert_eq!(fail.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
let OperationErrorKind::Step(step_err) = err.kind() else {
panic!("expected a step error, got {err:?}");
};
assert!(
matches!(step_err.kind(), StepErrorKind::Interrupted),
"got {step_err:?}"
);
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some(STEP_INTERRUPTED_ERROR_TYPE)
);
assert!(
!executed.load(Ordering::SeqCst),
"closure should NOT execute when AtMostOncePerRetry and replay=Started"
);
}
#[tokio::test]
async fn interrupted_attempt_gives_strategy_step_error_interrupted() {
let executed = Arc::new(std::sync::atomic::AtomicBool::new(false));
let (ctx, _client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
let seen_clone = Arc::clone(&seen);
let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
let source = std::error::Error::source(err).map(ToString::to_string);
seen_clone
.lock()
.expect("strategy log lock")
.push((err.kind().to_string(), source));
RetryDecision::Stop
});
let exec = at_most_once_over_started(ctx, Some(strategy), executed);
exec.execute()
.await
.expect_err("an interrupted attempt with no retry fails");
let expected = StepError::interrupted();
assert_eq!(
*seen.lock().expect("strategy log lock"),
[(
expected.kind().to_string(),
std::error::Error::source(&expected).map(ToString::to_string),
)]
);
}
#[tokio::test]
async fn step_at_most_once_replay_started_with_retry_suspends() {
use std::sync::atomic::{AtomicBool, Ordering};
let executed = Arc::new(AtomicBool::new(false));
let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
let (strategy, seen) =
kind_recording_strategy(RetryDecision::retry(Duration::from_secs(5)));
let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
let root = ctx.clone();
let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
let outcome =
crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
let retry = recorded_error_for(&client, &OperationAction::Retry);
assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
assert_eq!(retry.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
assert!(
!executed.load(Ordering::SeqCst),
"closure should NOT execute when AtMostOncePerRetry and replay=Started"
);
}
#[tokio::test]
async fn step_at_most_once_interrupted_default_strategy_retries() {
use std::sync::atomic::AtomicBool;
let executed = Arc::new(AtomicBool::new(false));
let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
let root = ctx.clone();
let exec = at_most_once_over_started(ctx, None, executed);
let outcome =
crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let retry = recorded_error_for(&client, &OperationAction::Retry);
assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
}
#[tokio::test]
async fn step_body_error_strategy_sees_execution_failed() {
let (ctx, client) = context_over(Vec::new());
let op_id = ctx.mint_id();
let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: Some(strategy),
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::AtMostOncePerRetry,
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("the step must fail");
assert_eq!(*seen.lock().expect("kind log lock"), ["execution failed"]);
let fail = recorded_error_for(&client, &OperationAction::Fail);
assert_eq!(fail.error_type(), Some(STEP_FALLBACK_ERROR_TYPE));
let OperationErrorKind::Step(step_err) = err.kind() else {
panic!("expected a step error, got {err:?}");
};
assert!(
matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
"got {step_err:?}"
);
}
#[tokio::test]
async fn step_interrupted_fail_record_replays_as_interrupted() {
let wire = crate::error::WireError::new(
Some(STEP_INTERRUPTED_ERROR_TYPE),
Some(STEP_INTERRUPTED_MESSAGE),
);
let (ctx, _client) =
context_over(vec![step_record(CheckpointStatus::Failed, 1, Some(&wire))]);
let op_id = ctx.mint_id();
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::AtMostOncePerRetry,
sub_type: OperationSubType::STEP,
plugin_name: None,
closure: |_: StepContext| async { Ok::<i32, BoxError>(1) },
_marker: std::marker::PhantomData,
};
let err = exec
.execute()
.await
.expect_err("the replayed step must fail");
let OperationErrorKind::Step(step_err) = err.kind() else {
panic!("expected a step error, got {err:?}");
};
assert!(
matches!(step_err.kind(), StepErrorKind::Interrupted),
"got {step_err:?}"
);
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some(STEP_INTERRUPTED_ERROR_TYPE)
);
let replayed = std::error::Error::source(step_err)
.and_then(|s| s.downcast_ref::<crate::error::ReplayedFailure>())
.expect("replay source is a ReplayedFailure");
assert_eq!(replayed.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
}
#[tokio::test]
async fn step_at_least_once_replay_started_re_executes() {
use crate::engine::{CheckpointLog, CheckpointRecord};
use std::sync::atomic::{AtomicBool, Ordering};
let executed = Arc::new(AtomicBool::new(false));
let executed_clone = Arc::clone(&executed);
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let closure = move |_: StepContext| {
executed_clone.store(true, Ordering::SeqCst);
async { Ok::<i32, BoxError>(77) }
};
let exec = StepExecution {
ctx,
op_id,
name: None,
retry_strategy: None,
serdes: crate::serdes::JsonSerdes,
semantics: StepSemantics::AtLeastOncePerRetry,
sub_type: OperationSubType::STEP,
plugin_name: None,
closure,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), 77);
assert!(
executed.load(Ordering::SeqCst),
"closure SHOULD execute under AtLeastOncePerRetry with Started replay"
);
}
use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
fn sample_error() -> StepError {
StepError::new(StepErrorKind::ExecutionFailed, Some("boom".into()))
}
fn retry_secs(decision: &RetryDecision) -> u64 {
match decision {
RetryDecision::Retry { delay } => delay.as_secs(),
RetryDecision::Stop => panic!("expected Retry, got Stop"),
}
}
#[test]
fn default_config_without_jitter_matches_documented_constants() {
let config = RetryStrategyConfig::builder()
.max_attempts(6)
.max_delay(Duration::from_mins(1))
.jitter(JitterStrategy::None)
.build();
let expected = [5u64, 10, 20, 40, 60];
for (attempt, expected_secs) in (1u32..).zip(expected) {
assert_eq!(
retry_secs(&config.decide(&sample_error(), attempt)),
expected_secs,
"attempt {attempt}"
);
}
assert_eq!(config.decide(&sample_error(), 6), RetryDecision::Stop);
assert_eq!(config.decide(&sample_error(), 7), RetryDecision::Stop);
}
#[test]
fn default_retry_strategy_matches_default_config_modulo_jitter() {
let strategy = default_retry_strategy();
let no_jitter = RetryStrategyConfig::builder()
.max_attempts(6)
.max_delay(Duration::from_mins(1))
.jitter(JitterStrategy::None)
.build();
let err = sample_error();
for attempt in 1u32..=5 {
let base = retry_secs(&no_jitter.decide(&sample_error(), attempt));
for _ in 0..50 {
let secs = retry_secs(&strategy(&err, attempt));
assert!(
(1..=base).contains(&secs),
"attempt {attempt}: delay {secs}s outside full-jitter envelope [1, {base}]"
);
}
}
assert_eq!(strategy(&err, 6), RetryDecision::Stop);
assert_eq!(no_jitter.decide(&sample_error(), 6), RetryDecision::Stop);
assert_eq!(strategy(&err, 7), RetryDecision::Stop);
}
#[test]
fn half_jitter_delays_stay_within_bounds() {
let config = RetryStrategyConfig::builder()
.max_attempts(6)
.initial_delay(Duration::from_secs(8))
.max_delay(Duration::from_secs(64))
.backoff_rate(2.0)
.jitter(JitterStrategy::Half)
.build();
for attempt in 1u32..=3 {
let base = 8u64 << (attempt - 1);
for _ in 0..100 {
let secs = retry_secs(&config.decide(&sample_error(), attempt));
assert!(
(base / 2..=base).contains(&secs),
"attempt {attempt}: delay {secs}s outside half-jitter bounds \
[{}, {base}]",
base / 2
);
}
}
}
#[test]
fn delays_are_capped_and_have_one_second_floor() {
let tiny = RetryStrategyConfig::builder()
.initial_delay(Duration::from_millis(1))
.jitter(JitterStrategy::None)
.build();
assert_eq!(retry_secs(&tiny.decide(&sample_error(), 1)), 1);
let capped = RetryStrategyConfig::builder()
.initial_delay(Duration::from_secs(30))
.max_delay(Duration::from_secs(45))
.backoff_rate(100.0)
.jitter(JitterStrategy::None)
.build();
assert_eq!(retry_secs(&capped.decide(&sample_error(), 1)), 30);
assert_eq!(retry_secs(&capped.decide(&sample_error(), 2)), 45);
}
#[test]
fn max_attempts_one_never_retries() {
let config = RetryStrategyConfig::builder().max_attempts(1).build();
assert_eq!(config.decide(&sample_error(), 1), RetryDecision::Stop);
}
#[test]
fn quantize_delay_rounds_to_nearest_with_one_second_floor() {
let cases = [
(0.0, 1u64),
(0.4, 1),
(1.1, 1),
(2.25, 2),
(2.5, 3),
(4.4, 4),
(5.0, 5),
(f64::NAN, 1),
(f64::INFINITY, 1),
];
for (input, expected) in cases {
assert_eq!(
quantize_delay(input),
Duration::from_secs(expected),
"input {input}"
);
}
}
#[test]
fn fractional_delays_round_to_nearest() {
let config = RetryStrategyConfig::builder()
.initial_delay(Duration::from_millis(1100))
.jitter(JitterStrategy::None)
.build();
assert_eq!(retry_secs(&config.decide(&sample_error(), 1)), 1);
assert_eq!(retry_secs(&config.decide(&sample_error(), 2)), 2);
let slow = RetryStrategyConfig::builder()
.max_attempts(6)
.initial_delay(Duration::from_secs(1))
.backoff_rate(1.5)
.jitter(JitterStrategy::None)
.build();
assert_eq!(retry_secs(&slow.decide(&sample_error(), 3)), 2);
let half = RetryStrategyConfig::builder()
.initial_delay(Duration::from_millis(2500))
.jitter(JitterStrategy::None)
.build();
assert_eq!(retry_secs(&half.decide(&sample_error(), 1)), 3);
}
#[test]
fn linear_backoff_cases() {
let linear = |initial: Duration, increment: Duration| {
RetryStrategyConfig::builder()
.initial_delay(initial)
.backoff(Backoff::linear(increment))
.jitter(JitterStrategy::None)
};
let delays = |config: &RetryStrategyConfig, attempts: std::ops::RangeInclusive<u32>| {
attempts
.map(|attempt| retry_secs(&config.decide(&sample_error(), attempt)))
.collect::<Vec<_>>()
};
let preset = RetryStrategyConfig::linear();
assert_eq!(delays(&preset, 1..=5), [1, 2, 3, 4, 5], "case A");
assert_eq!(
preset.decide(&sample_error(), 6),
RetryDecision::Stop,
"case B"
);
let capped = linear(Duration::from_secs(1), Duration::from_secs(10))
.max_delay(Duration::from_secs(15))
.max_attempts(6)
.build();
assert_eq!(delays(&capped, 1..=3), [1, 11, 15], "case C");
let fractional = linear(Duration::from_secs(1), Duration::from_millis(250))
.max_attempts(6)
.build();
assert_eq!(delays(&fractional, 1..=4), [1, 1, 2, 2], "case D");
let zero = linear(Duration::ZERO, Duration::ZERO)
.max_attempts(3)
.build();
assert_eq!(delays(&zero, 1..=1), [1], "case E");
assert_eq!(
RetryStrategyConfig::no_retry().decide(&sample_error(), 1),
RetryDecision::Stop,
"case F"
);
let exponential = RetryStrategyConfig::builder()
.max_attempts(6)
.max_delay(Duration::from_mins(1))
.jitter(JitterStrategy::None)
.build();
assert_eq!(delays(&exponential, 1..=5), [5, 10, 20, 40, 60], "case G");
}
#[test]
fn linear_half_jitter_stays_within_bounds() {
let config = RetryStrategyConfig::builder()
.initial_delay(Duration::from_secs(8))
.backoff(Backoff::linear(Duration::from_secs(8)))
.jitter(JitterStrategy::Half)
.build();
for (attempt, base) in [(1u32, 8u64), (2, 16), (3, 24)] {
for _ in 0..100 {
let secs = retry_secs(&config.decide(&sample_error(), attempt));
assert!(
(base / 2..=base).contains(&secs),
"attempt {attempt}: delay {secs}s outside half-jitter bounds \
[{}, {base}]",
base / 2
);
}
}
}
#[derive(Debug)]
struct Transient;
impl std::fmt::Display for Transient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("transient")
}
}
impl std::error::Error for Transient {}
fn is_transient(err: &StepError) -> bool {
std::error::Error::source(err)
.and_then(|e| e.downcast_ref::<Transient>())
.is_some()
}
fn mentions_timeout(err: &StepError) -> bool {
std::error::Error::source(err)
.map(ToString::to_string)
.is_some_and(|message| message.contains("timeout"))
}
fn body_error(source: crate::error::Source) -> StepError {
StepError::new(StepErrorKind::ExecutionFailed, Some(source))
}
#[test]
fn retry_if_filter_cases() {
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
type Predicate = fn(&StepError) -> bool;
type Case = (
&'static str,
Option<Predicate>,
StepError,
u32,
Option<u64>,
bool,
);
let either: Predicate = |err| is_transient(err) || mentions_timeout(err);
let boom = || body_error("boom".into());
let transient = || body_error(Box::new(Transient));
let timeout = || body_error("read timeout".into());
let cases: Vec<Case> = vec![
("A", None, boom(), 1, Some(1), false),
("B", Some(|_| false), boom(), 1, None, true),
("C", Some(|_| true), boom(), 2, Some(2), true),
("D", Some(|_| true), boom(), 3, None, false),
("E", Some(is_transient), transient(), 1, Some(1), true),
("F", Some(is_transient), boom(), 1, None, true),
("G", Some(mentions_timeout), timeout(), 1, Some(1), true),
("H", Some(either), boom(), 1, None, true),
("I", Some(either), timeout(), 1, Some(1), true),
];
for (case, predicate, err, attempt, expected, called) in cases {
let calls = Arc::new(AtomicU32::new(0));
let mut builder = RetryStrategyConfig::builder()
.max_attempts(3)
.initial_delay(Duration::from_secs(1))
.jitter(JitterStrategy::None);
if let Some(predicate) = predicate {
let calls = Arc::clone(&calls);
builder = builder.retry_if(move |err| {
calls.fetch_add(1, Ordering::SeqCst);
predicate(err)
});
}
let config = builder.build();
let decision = config.decide(&err, attempt);
let expected = expected.map_or(RetryDecision::Stop, |secs| {
RetryDecision::retry(Duration::from_secs(secs))
});
assert_eq!(decision, expected, "case {case}");
assert_eq!(
calls.load(Ordering::SeqCst),
u32::from(called),
"case {case}: predicate calls"
);
}
}
#[test]
fn retry_if_sees_the_error_kind() {
let config = RetryStrategyConfig::builder()
.max_attempts(3)
.initial_delay(Duration::from_secs(1))
.jitter(JitterStrategy::None)
.retry_if(|err| matches!(err.kind(), StepErrorKind::ExecutionFailed))
.build();
let interrupted = StepError::new(StepErrorKind::Interrupted, None);
assert_eq!(config.decide(&interrupted, 1), RetryDecision::Stop);
assert_eq!(
config.decide(&sample_error(), 1),
RetryDecision::retry(Duration::from_secs(1))
);
}
#[test]
fn into_retry_strategy_applies_retry_if() {
let strategy = RetryStrategyConfig::builder()
.retry_if(|_| false)
.build()
.into_retry_strategy();
assert_eq!(strategy(&sample_error(), 1), RetryDecision::Stop);
}
}
#[cfg(test)]
mod coordinator_tests {
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{
RefreshClient, RefreshRequest, assert_near, drive_handler, named, operation_like,
poll_once, refresh_ctx, wire_err,
};
use aws_sdk_lambda::types::{
ErrorObject, Operation, OperationAction, OperationStatus, OperationType, OperationUpdate,
StepDetails,
};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
fn secs(n: u64) -> Duration {
Duration::from_secs(n)
}
fn step_record(
req: &RefreshRequest<'_>,
update: &OperationUpdate,
status: OperationStatus,
next_in: Option<Duration>,
) -> Operation {
let mut details = StepDetails::builder().attempt(1);
if let Some(next_in) = next_in {
details = details.next_attempt_timestamp(req.timestamp_in(next_in));
}
operation_like(update, status)
.step_details(details.build())
.build()
.expect("all required fields set")
}
fn retry_client(name: &'static str) -> RefreshClient {
RefreshClient::new(move |req| {
let mut ops = Vec::new();
if let Some(retry) = named(req.updates, name, &OperationAction::Retry) {
ops.push(step_record(
req,
retry,
OperationStatus::Pending,
Some(secs(1)),
));
}
if req.empty_number == 1
&& let Some(retry) = named(req.history, name, &OperationAction::Retry)
{
ops.push(step_record(req, retry, OperationStatus::Ready, None));
}
Ok(ops)
})
}
fn retry_once(_err: &crate::StepError, attempt: u32) -> crate::RetryDecision {
if attempt < 2 {
crate::RetryDecision::retry(secs(1))
} else {
crate::RetryDecision::Stop
}
}
async fn sleeping_step(ctx: &crate::DurableContext) -> Result<u32, crate::error::WireError> {
ctx.step(|_| async {
tokio::time::sleep(secs(3)).await;
Ok(1_u32)
})
.name("s")
.await
.map_err(wire_err)
}
#[tokio::test(start_paused = true)]
async fn step_retry_runs_next_attempt_in_invocation() {
let client = Arc::new(retry_client("flaky"));
let ctx = refresh_ctx(&client);
let start = tokio::time::Instant::now();
let attempts = Arc::new(Mutex::new(Vec::new()));
let body_attempts = Arc::clone(&attempts);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(move |step_ctx| {
let attempts = Arc::clone(&body_attempts);
async move {
attempts
.lock()
.expect("lock")
.push((step_ctx.attempt(), start.elapsed()));
if step_ctx.attempt() < 2 {
return Err::<u32, crate::BoxError>("transient".into());
}
Ok(7)
}
})
.name("flaky")
.retry_strategy(retry_once)
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let value = flaky.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
let attempts = attempts.lock().expect("lock").clone();
let numbers: Vec<u32> = attempts.iter().map(|(n, _)| *n).collect();
assert_eq!(numbers, vec![1, 2]);
assert_near(attempts.get(1).expect("attempt 2 ran").1, secs(1));
}
#[tokio::test(start_paused = true)]
async fn step_retry_in_invocation_fires_start() {
use crate::plugin::AttemptOutcome::{Failed, Succeeded};
let client = Arc::new(retry_client("flaky"));
let hooks = crate::plugin::test_support::OpHooks::default();
let ctx = crate::DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
Duration::ZERO,
0,
Some(hooks.plugins()),
);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(|step_ctx| async move {
if step_ctx.attempt() < 2 {
return Err::<u32, crate::BoxError>("transient".into());
}
Ok(7)
})
.name("flaky")
.retry_strategy(retry_once)
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let value = flaky.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
let records = hooks.of("flaky");
let shapes: Vec<_> = records
.iter()
.map(crate::plugin::test_support::OpRecord::shape)
.collect();
assert_eq!(
shapes,
[
("start", false, None, None),
("attempt_start", false, Some(1), None),
("attempt_end", false, Some(1), Some(Failed)),
("start", true, None, None),
("attempt_start", false, Some(2), None),
("attempt_end", false, Some(2), Some(Succeeded)),
("end", false, None, None),
]
);
let error = records.get(2).and_then(|r| r.error.as_deref());
assert_eq!(error, Some("transient"));
let result = records.get(5).and_then(|r| r.result.as_deref());
assert_eq!(result, Some("7"));
let statuses: Vec<_> = records.iter().map(|r| r.status).collect();
assert_eq!(
statuses,
[
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Pending),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Succeeded),
Some(crate::OperationStatus::Succeeded),
]
);
}
#[tokio::test(start_paused = true)]
async fn attempt_info_shared_by_hooks() {
let client = Arc::new(retry_client("flaky"));
let hooks = crate::plugin::recorder::UfHooks::default();
let ctx = crate::DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
Duration::ZERO,
0,
Some(hooks.plugins()),
);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(|step_ctx| async move {
if step_ctx.attempt() < 2 {
return Err::<u32, crate::BoxError>("transient".into());
}
Ok(7)
})
.name("flaky")
.retry_strategy(retry_once)
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let value = flaky.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
let records = hooks.of("flaky");
let starts: Vec<_> = records.iter().filter(|r| r.hook == "start").collect();
assert_eq!(starts.len(), 2, "{records:?}");
assert!(starts.get(1).is_some_and(|r| r.replay), "{starts:?}");
let second = records
.iter()
.position(|r| r.hook == "attempt_start" && r.attempt == Some(2))
.expect("attempt 2 started");
let shared: Vec<_> = records
.get(second..)
.unwrap_or_default()
.iter()
.filter(|r| {
matches!(
r.hook,
"attempt_start" | "uf_start" | "wrap_enter" | "wrap_exit" | "uf_end"
)
})
.collect();
let hook_names: Vec<_> = shared.iter().map(|r| r.hook).collect();
assert_eq!(
hook_names,
[
"attempt_start",
"uf_start",
"wrap_enter",
"wrap_exit",
"uf_end"
]
);
let first = shared.first().expect("attempt_start");
for record in &shared {
assert_eq!(
record.info_addr, first.info_addr,
"{} shares the info",
record.hook
);
assert_eq!(record.status, Some(crate::OperationStatus::Started));
assert_eq!(record.info_attempt, Some(2));
assert!(!record.replay, "{}", record.hook);
assert!(record.has_start_timestamp, "{}", record.hook);
}
}
#[tokio::test(start_paused = true)]
async fn each_attempt_starts_from_original_captures() {
let client = Arc::new(retry_client("vec"));
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let v = vec![0_u32];
let mut step = std::pin::pin!(
ctx.step(move |step_ctx| async move {
let mut v = v;
v.push(step_ctx.attempt());
if step_ctx.attempt() < 2 {
return Err::<Vec<u32>, crate::BoxError>("transient".into());
}
Ok(v)
})
.name("vec")
.retry_strategy(retry_once)
.spawn()
);
poll_once(step.as_mut()).await;
sleeping_step(&ctx).await?;
let v = step.await.map_err(wire_err)?;
Ok(format!("{v:?}"))
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("[0, 2]".to_owned()));
}
#[tokio::test(start_paused = true)]
async fn arc_capture_is_shared_across_attempts_in_invocation() {
let client = Arc::new(retry_client("arc"));
let ctx = refresh_ctx(&client);
let counter = Arc::new(AtomicUsize::new(0));
let handler_counter = Arc::clone(&counter);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut step = std::pin::pin!(
ctx.step(move |step_ctx| async move {
handler_counter.fetch_add(1, Ordering::SeqCst);
if step_ctx.attempt() < 2 {
return Err::<u32, crate::BoxError>("transient".into());
}
Ok(1)
})
.name("arc")
.retry_strategy(retry_once)
.spawn()
);
poll_once(step.as_mut()).await;
sleeping_step(&ctx).await?;
step.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_eq!(counter.load(Ordering::SeqCst), 2);
}
#[tokio::test(start_paused = true)]
async fn lone_step_retry_suspends() {
let client = Arc::new(retry_client("lone"));
let ctx = refresh_ctx(&client);
let runs = Arc::new(AtomicUsize::new(0));
let body_runs = Arc::clone(&runs);
let outcome = drive_handler(ctx, None, move |ctx| async move {
ctx.step(move |_| async move {
body_runs.fetch_add(1, Ordering::SeqCst);
Err::<u32, crate::BoxError>("transient".into())
})
.name("lone")
.retry_strategy(retry_once)
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
assert!(client.empty_calls().is_empty(), "no refresh was sent");
}
#[tokio::test(start_paused = true)]
async fn replayed_pending_step_runs_next_attempt_in_invocation() {
let client = Arc::new(RefreshClient::new(|req| {
let mut ops = Vec::new();
if req.empty_number == 1 {
ops.push(
operation_like(&flaky_update(), OperationStatus::Ready)
.step_details(StepDetails::builder().attempt(1).build())
.build()
.expect("all required fields set"),
);
}
Ok(ops)
}));
let ctx = pending_flaky_ctx(&client);
let attempts = Arc::new(Mutex::new(Vec::new()));
let body_attempts = Arc::clone(&attempts);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(move |step_ctx| {
let attempts = Arc::clone(&body_attempts);
async move {
attempts.lock().expect("lock").push(step_ctx.attempt());
Ok::<u32, crate::BoxError>(7)
}
})
.name("flaky")
.retry_strategy(retry_once)
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let value = flaky.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
assert_eq!(*attempts.lock().expect("lock"), vec![2]);
}
fn pending_flaky_ctx(client: &Arc<RefreshClient>) -> crate::DurableContext {
let pending = operation_like(&flaky_update(), OperationStatus::Pending)
.step_details(
StepDetails::builder()
.attempt(1)
.next_attempt_timestamp(aws_smithy_types::DateTime::from(
std::time::SystemTime::now() + secs(1),
))
.build(),
)
.build()
.expect("all required fields set");
crate::DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::client::operations_to_checkpoint_log(&[pending])),
Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
)
}
#[tokio::test(start_paused = true)]
async fn pending_step_returns_terminal_record_in_invocation() {
let (events, _guard) = crate::driver::test_support::EventMessages::install();
let client = Arc::new(RefreshClient::new(|req| {
let mut ops = Vec::new();
if req.empty_number == 1 {
ops.push(
operation_like(&flaky_update(), OperationStatus::Succeeded)
.step_details(StepDetails::builder().attempt(2).result("5").build())
.build()
.expect("all required fields set"),
);
}
Ok(ops)
}));
let ctx = pending_flaky_ctx(&client);
let runs = Arc::new(AtomicUsize::new(0));
let body_runs = Arc::clone(&runs);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(move |_| async move {
body_runs.fetch_add(1, Ordering::SeqCst);
Ok::<u32, crate::BoxError>(7)
})
.name("flaky")
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let value = flaky.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("5".to_owned()));
assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
assert_eq!(
events.replayed(),
0,
"a resumed record emits no replay event"
);
}
#[tokio::test(start_paused = true)]
async fn pending_step_returns_failed_record_in_invocation() {
let (events, _guard) = crate::driver::test_support::EventMessages::install();
let client = Arc::new(RefreshClient::new(|req| {
let mut ops = Vec::new();
if req.empty_number == 1 {
ops.push(
operation_like(&flaky_update(), OperationStatus::Failed)
.step_details(
StepDetails::builder()
.attempt(2)
.error(
ErrorObject::builder()
.error_type("PaymentError")
.error_message("declined")
.build(),
)
.build(),
)
.build()
.expect("all required fields set"),
);
}
Ok(ops)
}));
let ctx = pending_flaky_ctx(&client);
let runs = Arc::new(AtomicUsize::new(0));
let body_runs = Arc::clone(&runs);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut flaky = std::pin::pin!(
ctx.step(move |_| async move {
body_runs.fetch_add(1, Ordering::SeqCst);
Ok::<u32, crate::BoxError>(7)
})
.name("flaky")
.spawn()
);
poll_once(flaky.as_mut()).await;
sleeping_step(&ctx).await?;
let err = flaky.await.expect_err("the recorded failure returns");
let error_type = err
.wire()
.and_then(crate::WireError::error_type)
.unwrap_or_default()
.to_owned();
Ok(error_type)
})
.await
.expect("no panic");
assert_eq!(
outcome,
InvocationOutcome::Complete("PaymentError".to_owned())
);
assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
assert_eq!(
events.replayed(),
0,
"a resumed record emits no replay event"
);
}
fn flaky_update() -> OperationUpdate {
OperationUpdate::builder()
.id(crate::engine::compute_wire_id_public("1"))
.r#type(OperationType::Step)
.sub_type(super::STEP_SUB_TYPE)
.name("flaky")
.action(OperationAction::Start)
.build()
.expect("all required fields set")
}
}
#[cfg(test)]
mod start_write_tests {
use crate::StepSemantics;
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{LatencyClient, drive_handler, latency_ctx, wire_err};
use aws_sdk_lambda::types::OperationAction;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
const LATENCY: Duration = Duration::from_millis(500);
fn ms(n: u64) -> Duration {
Duration::from_millis(n)
}
async fn timed_step(
buffer_window: Duration,
semantics: StepSemantics,
) -> (Arc<LatencyClient>, Duration, InvocationOutcome) {
let client = Arc::new(LatencyClient::new(LATENCY));
let ctx = latency_ctx(&client, buffer_window);
let started = Arc::new(Mutex::new(None));
let body_started = Arc::clone(&started);
let clock = Arc::clone(&client);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let body_clock = Arc::clone(&clock);
let value = ctx
.step(move |_| {
let started = Arc::clone(&body_started);
let clock = Arc::clone(&body_clock);
async move {
*started.lock().expect("lock") = Some(clock.elapsed());
Ok(1_u32)
}
})
.name("s")
.semantics(semantics)
.await
.map_err(wire_err)?;
Ok(format!("{value}@{}", clock.elapsed().as_millis()))
})
.await
.expect("no panic");
let started = started.lock().expect("lock").expect("the body ran");
(client, started, outcome)
}
#[tokio::test(start_paused = true)]
async fn at_least_once_body_starts_before_start_response() {
let (client, started, outcome) =
timed_step(Duration::ZERO, StepSemantics::AtLeastOncePerRetry).await;
assert_eq!(started, ms(0), "the body starts before START is answered");
let start = client.request_with(&OperationAction::Start).expect("START");
let succeed = client
.request_with(&OperationAction::Succeed)
.expect("SUCCEED");
assert_eq!(start.received, ms(0));
assert_eq!(succeed.received, ms(500), "SUCCEED waits for START");
assert_eq!(outcome, InvocationOutcome::Complete("1@1000".to_owned()));
}
#[tokio::test(start_paused = true)]
async fn buffered_start_precedes_outcome() {
let (client, started, outcome) =
timed_step(ms(50), StepSemantics::AtLeastOncePerRetry).await;
assert_eq!(started, ms(0), "the body starts before START is answered");
let requests = client.requests();
let first = requests.first().expect("a request");
let actions: Vec<&OperationAction> = first
.updates
.iter()
.map(aws_sdk_lambda::types::OperationUpdate::action)
.collect();
assert_eq!(
actions,
vec![&OperationAction::Start],
"the first request carries only START"
);
assert!(
requests.iter().skip(1).any(|r| r
.updates
.iter()
.any(|u| u.action() == &OperationAction::Succeed)),
"SUCCEED is in a later request"
);
assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
}
#[tokio::test(start_paused = true)]
async fn at_most_once_body_waits_for_start_response() {
let (_client, started, outcome) =
timed_step(Duration::ZERO, StepSemantics::AtMostOncePerRetry).await;
assert_eq!(started, ms(500), "the body starts after START is answered");
assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
}
fn assert_failed_without_outcome(client: &LatencyClient, outcome: &InvocationOutcome) {
let InvocationOutcome::Failed { error } = outcome else {
unreachable!("expected a failed execution, got {outcome:?}");
};
assert_eq!(
error.error_type(),
Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
assert!(client.request_with(&OperationAction::Start).is_some());
assert!(!client.received_outcome(), "no outcome update was sent");
}
#[tokio::test(start_paused = true)]
async fn failed_start_write_writes_no_outcome() {
let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
let runs = Arc::new(AtomicUsize::new(0));
let body_runs = Arc::clone(&runs);
let outcome = drive_handler(
latency_ctx(&client, Duration::ZERO),
None,
move |ctx| async move {
ctx.step(move |_| async move {
body_runs.fetch_add(1, Ordering::SeqCst);
Ok(1_u32)
})
.name("s")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
},
)
.await
.expect("no panic");
assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
assert_failed_without_outcome(&client, &outcome);
let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
let checks = Arc::new(AtomicUsize::new(0));
let check_runs = Arc::clone(&checks);
let outcome = drive_handler(
latency_ctx(&client, Duration::ZERO),
None,
move |ctx| async move {
ctx.wait_for_condition(
move |_, state: u32| {
let checks = Arc::clone(&check_runs);
async move {
checks.fetch_add(1, Ordering::SeqCst);
Ok(state + 1)
}
},
0,
)
.wait_strategy_fn(|_state, _attempt| {
crate::builders::wait_for_condition::WaitDecision::complete()
})
.name("c")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
},
)
.await
.expect("no panic");
assert_eq!(checks.load(Ordering::SeqCst), 1, "the check ran once");
assert_failed_without_outcome(&client, &outcome);
}
fn assert_start_kept(
client: &LatencyClient,
ctx: &crate::DurableContext,
result: Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>,
returned: Duration,
) {
let start = client.request_with(&OperationAction::Start).expect("START");
let answered = start.answered.expect("START was answered");
assert_eq!(answered, start.received + LATENCY);
assert!(!client.received_outcome(), "no outcome update was sent");
let wire_id = start.updates.first().expect("an update").id();
assert_eq!(crate::engine::compute_wire_id_public("1"), wire_id);
let view = ctx.checkpoint_status_view("1").expect("a record");
assert_eq!(view.status, crate::engine::CheckpointStatus::Started);
let payload = result.expect_err("the panic leaves the drive");
assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
assert!(
returned >= answered,
"the drive returned at {returned:?}, before {answered:?}"
);
}
async fn panicking_step(buffer_window: Duration) {
let client = Arc::new(LatencyClient::new(LATENCY));
let ctx = latency_ctx(&client, buffer_window);
let clock = Arc::clone(&client);
let result = drive_handler(ctx.clone(), None, |ctx| async move {
#[expect(clippy::panic)] let (): () = ctx
.step(move |_| async move {
assert_eq!(
clock.elapsed(),
ms(0),
"the body runs before START is answered"
);
panic!("boom")
})
.name("s")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await;
assert_start_kept(&client, &ctx, result, client.elapsed());
}
#[tokio::test(start_paused = true)]
async fn body_panic_before_start_response_keeps_start_write() {
panicking_step(Duration::ZERO).await;
panicking_step(ms(50)).await;
let client = Arc::new(LatencyClient::new(LATENCY));
let ctx = latency_ctx(&client, Duration::ZERO);
let clock = Arc::clone(&client);
let result = drive_handler(ctx.clone(), None, |ctx| async move {
#[expect(clippy::panic)] ctx.wait_for_condition(
move |_, _state: u32| {
let clock = Arc::clone(&clock);
async move {
assert_eq!(
clock.elapsed(),
ms(0),
"the check runs before START is answered"
);
panic!("boom")
}
},
0,
)
.wait_strategy_fn(|_state, _attempt| {
crate::builders::wait_for_condition::WaitDecision::complete()
})
.name("c")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await;
assert_start_kept(&client, &ctx, result, client.elapsed());
}
}