use std::future::Future;
use std::time::Duration;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use crate::context::{DurableContext, StepContext, SubmittedWrite};
use crate::engine::CheckpointStatus;
use crate::engine::OperationId;
use crate::error::{
OperationError, OperationErrorKind, WaitForConditionError, WaitForConditionErrorKind,
};
use crate::plugin::UserBody;
use crate::serdes::SerdesContext;
use crate::{BoxError, Serdes};
pub(crate) const WFC_SUB_TYPE: &str = "WaitForCondition";
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum WaitDecision {
Complete,
#[non_exhaustive]
Continue {
delay: Duration,
},
#[non_exhaustive]
Exhausted {
reason: String,
},
}
impl WaitDecision {
#[must_use]
pub fn complete() -> Self {
Self::Complete
}
#[must_use]
pub fn continue_with(delay: Duration) -> Self {
Self::Continue { delay }
}
#[must_use]
pub fn exhausted(reason: impl Into<String>) -> Self {
Self::Exhausted {
reason: reason.into(),
}
}
}
pub(crate) type WaitStrategyFn<S> = Box<dyn Fn(S, u32) -> WaitDecision + Send + Sync>;
pub(crate) struct WaitForConditionExecution<S, F, SD> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) initial_state: S,
pub(crate) wait_strategy: WaitStrategyFn<S>,
pub(crate) serdes: SD,
pub(crate) check: F,
}
impl<S, F, Fut, SD> WaitForConditionExecution<S, F, SD>
where
S: Clone + Send + Sync + 'static,
F: Fn(StepContext, S) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<S, BoxError>> + Send + 'static,
SD: Serdes<S>,
{
pub(crate) async fn execute(self) -> Result<S, OperationError> {
let Self {
ctx,
op_id,
name,
initial_state,
wait_strategy,
serdes,
check,
} = self;
let core = WfcCore {
ctx,
op_id,
name,
initial_state,
wait_strategy,
serdes,
};
let mut resumed = false;
loop {
match core.before(resumed).await? {
WfcPrelude::Done(result) => return result,
WfcPrelude::Wait => {}
WfcPrelude::Run {
attempt,
state,
start,
info,
} => {
let body = UserBody::Step(core.op_id.positional().to_owned());
let step_ctx = StepContext::new(attempt);
let span = crate::tracing_layer::operation_span(
core.ctx.execution_arn(),
&core.ctx.lambda_context().request_id,
core.ctx.lambda_context().tenant_id.as_deref(),
core.op_id.wire(),
core.name.as_deref(),
attempt,
false,
);
let check = ✓
let ran = core
.ctx
.hold_activity(crate::plugin::run_user_function(
core.ctx.shared_plugins(),
info,
body,
span,
move || (check)(step_ctx, state),
|error: &BoxError| crate::error::wire_error_for(&**error, "Error"),
))
.await;
core.await_start(start).await?;
match core.settle(attempt, ran).await? {
WfcSettled::Done(value) => return Ok(value),
WfcSettled::Retry => {}
}
}
}
core.ctx
.wait_for_record(core.op_id.wire(), |status| {
*status != CheckpointStatus::Pending
})
.await;
resumed = true;
}
}
}
struct WfcCore<S, SD> {
ctx: DurableContext,
op_id: OperationId,
name: Option<String>,
initial_state: S,
wait_strategy: WaitStrategyFn<S>,
serdes: SD,
}
enum WfcPrelude<S> {
Done(Result<S, OperationError>),
Wait,
Run {
attempt: u32,
state: S,
start: Option<SubmittedWrite>,
info: std::sync::Arc<crate::plugin::OperationInfo>,
},
}
enum WfcSettled<S> {
Done(S),
Retry,
}
impl<S, SD> WfcCore<S, SD>
where
S: Clone + Send + Sync + 'static,
SD: Serdes<S>,
{
#[expect(clippy::too_many_lines)] async fn before(&self, resumed: bool) -> Result<WfcPrelude<S>, OperationError> {
self.ctx.enforce_task_ownership()?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let serdes_ctx = SerdesContext::new(&wire_id, self.ctx.execution_arn());
let mut already_started = false;
if let Some(view) = self.ctx.checkpoint_view_validated(
&positional_id,
&wire_id,
"Step",
Some(WFC_SUB_TYPE),
self.name.as_deref(),
)? {
match view.status {
CheckpointStatus::Succeeded => {
let payload = self.ctx.checkpoint_result_payload(&positional_id);
let value = replay_terminal_success(
&self.ctx,
self.name.as_deref(),
&self.serdes,
payload,
serdes_ctx.clone(),
)
.await?;
if !resumed {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Step",
Some(WFC_SUB_TYPE),
view.attempt,
);
}
return Ok(WfcPrelude::Done(Ok(value)));
}
CheckpointStatus::Failed => {
if !resumed {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Step",
Some(WFC_SUB_TYPE),
view.attempt,
);
}
let wire = self
.ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Ok(WfcPrelude::Done(Err(replay_terminal_failure(
wire,
&wire_id,
view.attempt,
))));
}
CheckpointStatus::Pending => {
return Ok(WfcPrelude::Wait);
}
CheckpointStatus::Started => {
already_started = true;
}
CheckpointStatus::Ready
| CheckpointStatus::Cancelled
| CheckpointStatus::TimedOut
| CheckpointStatus::Stopped => {
}
CheckpointStatus::Unknown(ref raw) => {
return Err(self.ctx.unrecognized_status_error(&wire_id, raw));
}
}
}
let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
let current_state =
if let Some(payload) = self.ctx.checkpoint_result_payload(&positional_id) {
deserialize_state_str(
&self.ctx,
self.name.as_deref(),
&self.serdes,
payload,
serdes_ctx.clone(),
)
.await?
} else {
let serialized = serialize_state(
&self.ctx,
self.name.as_deref(),
&self.serdes,
self.initial_state.clone(),
serdes_ctx.clone(),
)
.await?;
deserialize_state_str(
&self.ctx,
self.name.as_deref(),
&self.serdes,
serialized,
serdes_ctx.clone(),
)
.await?
};
let (start, info) = if already_started {
let info = match self.ctx.plugins() {
Some(plugins) => {
crate::plugin::attempt_started(plugins, self.info(), attempt, true)
}
None => std::sync::Arc::new(self.info()),
};
(None, info)
} else {
let start_update = build_wfc_update(
&wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
OperationAction::Start,
None,
None,
);
let reserved = self.ctx.reserve_checkpoint_updates(vec![start_update]);
let plugins = self.ctx.plugins();
if let Some(plugins) = plugins {
let info = self
.info()
.with_status(crate::OperationStatus::Started)
.with_replay(attempt > 1);
crate::plugin::operation_started(plugins, &info);
}
let start = reserved.release();
let info = match plugins {
Some(plugins) => crate::plugin::attempt_start_only(plugins, self.info(), attempt),
None => std::sync::Arc::new(self.info()),
};
(Some(start), info)
};
Ok(WfcPrelude::Run {
attempt,
state: current_state,
start,
info,
})
}
async fn await_start(&self, start: Option<SubmittedWrite>) -> Result<(), OperationError> {
if let Some(start) = start
&& let Err(err) = start.await
{
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
Ok(())
}
#[expect(clippy::too_many_lines)] async fn settle(
&self,
attempt: u32,
ran: crate::plugin::Ran<S>,
) -> Result<WfcSettled<S>, OperationError> {
let wire_id = self.op_id.wire();
let serdes_ctx = SerdesContext::new(wire_id, self.ctx.execution_arn());
match ran.into_parts() {
Ok(new_state) => {
let round_trip = async {
let serialized = serialize_state(
&self.ctx,
self.name.as_deref(),
&self.serdes,
new_state,
serdes_ctx.clone(),
)
.await?;
let deserialized: S = deserialize_state_str(
&self.ctx,
self.name.as_deref(),
&self.serdes,
serialized.clone(),
serdes_ctx,
)
.await?;
Ok::<_, OperationError>((serialized, deserialized))
};
let (serialized, deserialized) = match round_trip.await {
Ok(pair) => pair,
Err(op_err) => {
let wire = crate::error::serialization_failure_wire(&op_err);
let update = build_wfc_fail_update(
wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
&wire,
);
if let Err(client_err) = self.ctx.checkpoint_updates(vec![update]).await {
return self.ctx.checkpoint_failure_unrecoverable(client_err).await;
}
return Err(op_err
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire));
}
};
let decision = (self.wait_strategy)(deserialized.clone(), attempt);
match decision {
WaitDecision::Complete => {
let update = build_wfc_update(
wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
OperationAction::Succeed,
Some(&serialized),
None,
);
if let Err(err) = self.ctx.checkpoint_updates(vec![update]).await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
self.check_ended(
attempt,
crate::OperationStatus::Succeeded,
Some(serialized),
None,
);
Ok(WfcSettled::Done(deserialized))
}
WaitDecision::Continue { delay } => {
#[expect(clippy::cast_possible_truncation)] let delay_secs = (delay.as_secs_f64().ceil() as i64)
.clamp(1, i64::from(i32::MAX))
as i32;
let update = build_wfc_update(
wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
OperationAction::Retry,
Some(&serialized),
Some(delay_secs),
);
if let Err(err) = self.ctx.checkpoint_updates(vec![update]).await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
self.check_ended(
attempt,
crate::OperationStatus::Pending,
Some(serialized),
None,
);
Ok(WfcSettled::Retry)
}
WaitDecision::Exhausted { reason } => {
let wire = crate::error::WireError::new(
Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE),
Some(reason.as_str()),
);
let update = build_wfc_fail_update(
wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
&wire,
);
if let Err(err) = self.ctx.checkpoint_updates(vec![update]).await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
self.check_ended(
attempt,
crate::OperationStatus::Failed,
None,
Some(&wire),
);
Err(wfc_op_error(WaitForConditionErrorKind::MaxChecksExceeded(
crate::error::MaxChecksExceeded::new(attempt),
))
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire))
}
}
}
Err((check_err, wire)) => {
let update = build_wfc_fail_update(
wire_id,
self.name.as_deref(),
self.ctx.parent_wire_id(),
&wire,
);
if let Err(err) = self.ctx.checkpoint_updates(vec![update]).await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
self.check_ended(attempt, crate::OperationStatus::Failed, None, Some(&wire));
Err(wfc_failure_error(wire, wire_id, attempt, check_err))
}
}
}
fn info(&self) -> crate::plugin::OperationInfo {
crate::plugin::OperationInfo::for_operation(
&self.ctx,
self.op_id.wire(),
crate::OperationKind::Step,
WFC_SUB_TYPE,
self.name.as_deref(),
)
}
fn check_ended(
&self,
attempt: u32,
status: crate::OperationStatus,
result: Option<String>,
error: Option<&crate::error::WireError>,
) {
let Some(plugins) = self.ctx.plugins() else {
return;
};
let outcome = if error.is_some() {
crate::plugin::AttemptOutcome::Failed
} else {
crate::plugin::AttemptOutcome::Succeeded
};
crate::plugin::attempt_ended(
plugins,
self.info()
.with_status(status)
.with_result(result)
.with_error(error.cloned()),
crate::plugin::AttemptEnd {
attempt,
outcome,
error: error.cloned(),
ends_operation: status != crate::OperationStatus::Pending,
},
);
}
}
async fn replay_terminal_success<S, SD: Serdes<S>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &SD,
result: Option<String>,
serdes_ctx: SerdesContext,
) -> Result<S, OperationError> {
let payload = result.unwrap_or_else(|| "null".to_owned());
deserialize_state_str(ctx, name, serdes, payload, serdes_ctx).await
}
fn replay_terminal_failure(
wire: crate::error::WireError,
wire_id: &str,
recorded_attempt: u32,
) -> OperationError {
let source = crate::error::ReplayedFailure::source_from(wire.clone());
wfc_failure_error(wire, wire_id, recorded_attempt.saturating_add(1), source)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WfcFailureClass {
MaxChecksExceeded,
SerializationFailed,
CheckFailed,
}
fn classify_wfc_failure(wire_type: Option<&str>) -> WfcFailureClass {
match wire_type {
Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE) => WfcFailureClass::MaxChecksExceeded,
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE) => WfcFailureClass::SerializationFailed,
_ => WfcFailureClass::CheckFailed,
}
}
fn wfc_failure_error(
wire: crate::error::WireError,
wire_id: &str,
checks: u32,
source: crate::error::Source,
) -> OperationError {
let err = match classify_wfc_failure(wire.error_type()) {
WfcFailureClass::MaxChecksExceeded => {
wfc_op_error(WaitForConditionErrorKind::MaxChecksExceeded(
crate::error::MaxChecksExceeded::new(checks),
))
}
WfcFailureClass::SerializationFailed => {
wfc_op_error_with_source(WaitForConditionErrorKind::SerializationFailed, Some(source))
}
WfcFailureClass::CheckFailed => {
wfc_op_error_with_source(WaitForConditionErrorKind::CheckFailed, Some(source))
}
};
err.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire)
}
async fn serialize_state<S, SD: Serdes<S>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &SD,
value: S,
serdes_ctx: SerdesContext,
) -> Result<String, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx.hold_activity(serdes.serialize(value, serdes_ctx)).await {
Ok(serialized) => Ok(serialized),
Err(err) => {
let e = ctx
.route_serdes_error("serialize", &operation_id, name, err)
.await;
Err(wfc_op_error_with_source(
WaitForConditionErrorKind::SerializationFailed,
Some(e),
))
}
}
}
async fn deserialize_state_str<S, SD: Serdes<S>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &SD,
payload: String,
serdes_ctx: SerdesContext,
) -> Result<S, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx
.hold_activity(serdes.deserialize(payload, serdes_ctx))
.await
{
Ok(value) => Ok(value),
Err(err) => {
let e = ctx
.route_serdes_error("deserialize", &operation_id, name, err)
.await;
Err(wfc_op_error_with_source(
WaitForConditionErrorKind::SerializationFailed,
Some(crate::error::ContextualError::source_from(
"state deserialization failed",
e,
)),
))
}
}
}
fn build_wfc_update(
wire_id: &str,
name: Option<&str>,
parent_wire_id: Option<&str>,
action: OperationAction,
payload: Option<&str>,
delay_secs: Option<i32>,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(WFC_SUB_TYPE)
.action(action);
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
if let Some(p) = payload {
builder = builder.payload(p);
}
if let Some(d) = delay_secs {
builder = builder.step_options(
aws_sdk_lambda::types::StepOptions::builder()
.next_attempt_delay_seconds(d)
.build(),
);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_wfc_fail_update(
wire_id: &str,
name: Option<&str>,
parent_wire_id: Option<&str>,
error: &crate::error::WireError,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id)
.r#type(OperationType::Step)
.sub_type(WFC_SUB_TYPE)
.action(OperationAction::Fail)
.error(error.to_error_object());
if let Some(n) = name {
builder = builder.name(n);
}
if let Some(parent) = parent_wire_id {
builder = builder.parent_id(parent);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn wfc_op_error(kind: WaitForConditionErrorKind) -> OperationError {
wfc_op_error_with_source(kind, None)
}
fn wfc_op_error_with_source(
kind: WaitForConditionErrorKind,
source: Option<crate::error::Source>,
) -> OperationError {
OperationError::from_kind(OperationErrorKind::WaitForCondition(
WaitForConditionError::new(kind, source),
))
}
#[cfg(test)]
#[expect(clippy::panic)] mod tests {
use super::*;
use crate::client::InMemoryExecutionClient;
use crate::context::DurableContext;
use crate::engine::{CheckpointLog, CheckpointRecord};
use std::sync::Arc;
fn make_ctx_with_client() -> DurableContext {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::empty());
DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
)
}
fn make_ctx_with_log(records: Vec<(String, CheckpointRecord)>) -> DurableContext {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let log = Arc::new(CheckpointLog::from_records(records));
DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
)
}
#[tokio::test]
async fn regression_carried_state_survives_start_checkpoint_merge() {
use aws_sdk_lambda::types::{Operation, OperationStatus, OperationType, StepDetails};
use std::sync::Mutex as StdMutex;
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Ready,
result: Some("1".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 1,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let start_response_op = Operation::builder()
.id(wire_key.clone())
.r#type(OperationType::Step)
.status(OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.step_details(StepDetails::builder().attempt(1).build())
.build()
.expect("all required Operation fields set");
client.enqueue_checkpoint_response(crate::client::TestResponse::Success(vec![
start_response_op,
]));
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let seen = Arc::new(StdMutex::new(None::<i32>));
let seen_check = Arc::clone(&seen);
let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
let root = ctx.clone();
let exec = WaitForConditionExecution {
ctx,
op_id,
name: None,
initial_state: 0i32,
wait_strategy: Box::new(|state: i32, _attempt| {
if state >= 3 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
}),
serdes: crate::serdes::JsonSerdes,
check: move |_ctx, state: i32| {
*seen_check
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(state);
async move { Ok::<i32, BoxError>(state + 1) }
},
};
let outcome =
crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let observed = seen
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.expect("check function must run on the Ready re-invoke path");
assert_eq!(
observed, 1,
"check must receive the carried state (1) from the pre-START \
checkpoint record, not initial_state (0); a START-merge reset \
would stall the condition in an unbounded retry loop"
);
}
#[tokio::test]
async fn failed_retry_checkpoint_does_not_mutate_prior_filesystem_state() {
use crate::client::TestResponse;
use crate::serdes::{FileSystemSerdes, Serdes, SerdesContext};
use std::time::{SystemTime, UNIX_EPOCH};
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let tmp = std::env::temp_dir().join(format!(
"wfc_failed_retry_checkpoint_{}_{}",
std::process::id(),
unique
));
let filesystem_serdes = FileSystemSerdes::new(tmp.to_string_lossy().into_owned());
let wire_key = crate::engine::compute_wire_id_public("1");
let serdes_ctx = SerdesContext::new(&wire_key, "arn:test");
let accepted_envelope = filesystem_serdes
.serialize(1_i32, serdes_ctx.clone())
.await
.unwrap();
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Ready,
result: Some(accepted_envelope.clone()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 1,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(TestResponse::Success(Vec::new()));
client.enqueue_checkpoint_response(TestResponse::NonRetryableError(
"injected Retry checkpoint failure".to_owned(),
));
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token0".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution {
ctx,
op_id,
name: None,
initial_state: 0i32,
wait_strategy: Box::new(|_state: i32, _attempt| {
WaitDecision::continue_with(Duration::from_secs(1))
}),
serdes: filesystem_serdes.clone(),
check: |_ctx, state: i32| async move { Ok::<i32, BoxError>(state + 1) },
};
let signal = Arc::clone(exec.ctx.coordinator());
let parked = tokio::time::timeout(Duration::from_millis(200), exec.execute()).await;
assert!(
parked.is_err(),
"a rejected RETRY checkpoint must park the future for the \
driver, not resolve"
);
let fatal = signal
.fatal_error()
.expect("the checkpoint failure records an execution-fatal error");
assert!(
fatal
.error_message
.contains("injected Retry checkpoint failure"),
"unexpected fatal: {fatal:?}"
);
let restored: i32 = filesystem_serdes
.deserialize(accepted_envelope, serdes_ctx)
.await
.unwrap();
let _ = std::fs::remove_dir_all(&tmp);
assert_eq!(
restored, 1,
"a failed Retry checkpoint must not mutate the state referenced \
by the prior accepted checkpoint"
);
}
#[tokio::test]
async fn condition_met_first_try() {
let ctx = make_ctx_with_client();
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution {
ctx,
op_id,
name: Some("immediate".to_owned()),
initial_state: 10i32,
wait_strategy: Box::new(|state: i32, _attempt| {
if state >= 5 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
}),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, state: i32| async move { Ok::<i32, BoxError>(state) },
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), 10);
}
#[tokio::test]
async fn condition_met_after_n_state_evolution() {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Ready,
result: Some("1".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 1,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let ctx = make_ctx_with_log(vec![(wire_key, record)]);
let op_id = ctx.mint_id(); let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
let root = ctx.clone();
let exec = WaitForConditionExecution {
ctx,
op_id,
name: None,
initial_state: 0i32,
wait_strategy: Box::new(|state: i32, _attempt| {
if state >= 3 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
}),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, state: i32| async move { Ok::<i32, BoxError>(state + 1) },
};
let outcome =
crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
}
#[tokio::test]
async fn exhaustion_produces_error() {
let ctx = make_ctx_with_client();
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution {
ctx,
op_id,
name: None,
initial_state: 0i32,
wait_strategy: Box::new(|_state: i32, attempt| {
if attempt >= 1 {
WaitDecision::exhausted("max attempts exceeded")
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
}),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, state: i32| async move { Ok::<i32, BoxError>(state + 1) },
};
let result = exec.execute().await;
assert!(result.is_err());
let err = result.unwrap_err();
match err.kind() {
OperationErrorKind::WaitForCondition(wfc_err) => match wfc_err.kind() {
WaitForConditionErrorKind::MaxChecksExceeded(details) => {
assert_eq!(details.checks(), 1);
}
other => panic!("expected MaxChecksExceeded, got: {other:?}"),
},
other => panic!("expected WaitForCondition error, got: {other:?}"),
}
let wire = err.wire().unwrap();
assert_eq!(
wire.error_type(),
Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE)
);
assert_eq!(wire.error_message(), Some("max attempts exceeded"));
assert!(
wire.stack_trace().is_empty(),
"the SDK captures no stack trace for an exhaustion"
);
}
#[tokio::test]
async fn regression_python574_corrupt_state_loud_error_not_silent_reset() {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Succeeded,
result: Some("\"not-a-number\"".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 2,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let ctx = make_ctx_with_log(vec![(wire_key, record)]);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: Some("regression".to_owned()),
initial_state: 0,
wait_strategy: Box::new(|_state: i32, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
#[expect(unreachable_code)] {
panic!("check must NOT execute during replay");
Ok::<i32, BoxError>(0)
}
},
};
let result = exec.execute().await;
assert!(
result.is_err(),
"corrupt state MUST produce an error, not silently reset to initial_state"
);
let err = result.unwrap_err();
match err.kind() {
OperationErrorKind::WaitForCondition(wfc_err) => match wfc_err.kind() {
WaitForConditionErrorKind::SerializationFailed => {
let message = crate::error::chain_string(wfc_err);
assert!(
message.contains("state deserialization failed"),
"error chain should indicate state deserialization failure: {message}"
);
}
other => panic!("expected SerializationFailed, got: {other:?}"),
},
other => panic!("expected WaitForCondition error, got: {other:?}"),
}
}
fn uf_ctx(
hooks: &crate::plugin::recorder::UfHooks,
records: Vec<(String, CheckpointRecord)>,
) -> (DurableContext, Arc<InMemoryExecutionClient>) {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
None,
0,
Some(hooks.plugins()),
);
(ctx, client)
}
fn recorded_fail(client: &InMemoryExecutionClient) -> (Option<String>, Option<String>) {
let updates = client.recorded_updates();
let fail = updates
.iter()
.find(|u| u.action() == &OperationAction::Fail)
.expect("a FAIL update");
let error = fail.error().expect("the FAIL carries an error");
(
error.error_type().map(str::to_owned),
error.error_message().map(str::to_owned),
)
}
const LIVE_CHECK: [&str; 6] = [
"attempt_start",
"uf_start",
"wrap_enter",
"wrap_exit",
"uf_end",
"attempt_end",
];
#[tokio::test]
async fn condition_check_error_and_exhaustion() {
use crate::builders::wait_for_condition::WaitStrategy;
use crate::plugin::recorder::UfHooks;
let hooks = UfHooks::default();
let (ctx, client) = uf_ctx(&hooks, Vec::new());
let err = ctx
.wait_for_condition(
|_ctx, _state: u32| async { Err::<u32, BoxError>("bad".into()) },
0_u32,
)
.wait_strategy_fn(|_state, _attempt| WaitDecision::complete())
.name("cond")
.await
.unwrap_err();
assert!(err.wire().is_some());
let mut expected = vec!["start"];
expected.extend(LIVE_CHECK);
expected.push("end");
assert_eq!(hooks.hooks_of("cond"), expected);
let end = hooks.find("cond", "uf_end");
assert_eq!(end.ok, Some(false));
let wire = end.wire.expect("an Err outcome");
assert_eq!(wire.error_type(), Some("Error"));
assert_eq!(wire.error_message(), Some("bad"));
assert_eq!(err.wire(), Some(&wire));
assert_eq!(
recorded_fail(&client),
(
wire.error_type().map(str::to_owned),
wire.error_message().map(str::to_owned)
)
);
let attempt_end = hooks.find("cond", "attempt_end");
assert_eq!(
attempt_end
.wire
.as_ref()
.and_then(crate::error::WireError::error_message),
Some("bad")
);
assert_eq!(attempt_end.status, Some(crate::OperationStatus::Failed));
assert_eq!(attempt_end.result, None);
let strategy = || {
WaitStrategy::builder(|_state: &u32| false)
.max_attempts(2)
.build()
};
let hooks = UfHooks::default();
let (ctx, client) = uf_ctx(&hooks, Vec::new());
let cond = ctx
.wait_for_condition(|_ctx, state: u32| async move { Ok(state + 1) }, 0_u32)
.wait_strategy(strategy())
.name("cond");
let signal = Arc::clone(ctx.coordinator());
let ownership = Arc::clone(ctx.task_ownership());
let outcome =
crate::driver::test_support::outcome_of(ctx.clone(), ownership, signal, cond).await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let mut expected = vec!["start"];
expected.extend(LIVE_CHECK);
assert_eq!(hooks.hooks_of("cond"), expected);
assert_eq!(hooks.find("cond", "uf_end").ok, Some(true));
assert!(
client
.recorded_updates()
.iter()
.any(|u| u.action() == &OperationAction::Retry)
);
let hooks = UfHooks::default();
let record = wfc_record(CheckpointStatus::Ready, Some("1"), None, 1);
let (ctx, client) = uf_ctx(&hooks, vec![record]);
let err = ctx
.wait_for_condition(|_ctx, state: u32| async move { Ok(state + 1) }, 0_u32)
.wait_strategy(strategy())
.name("cond")
.await
.unwrap_err();
assert!(err.wire().is_some());
let mut expected = vec!["start"];
expected.extend(LIVE_CHECK);
expected.push("end");
assert_eq!(hooks.hooks_of("cond"), expected);
let end = hooks.find("cond", "uf_end");
assert_eq!((end.ok, end.info_attempt), (Some(true), Some(2)));
let (error_type, _) = recorded_fail(&client);
assert_eq!(
error_type.as_deref(),
Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE)
);
}
#[tokio::test]
async fn condition_check_typed_error_reaches_the_end_hook() {
use crate::plugin::recorder::UfHooks;
let hooks = UfHooks::default();
let (ctx, client) = uf_ctx(&hooks, Vec::new());
let err = ctx
.wait_for_condition(
|_ctx, _state: u32| async {
Err::<u32, BoxError>(Box::new(crate::TypedError::with_type(
"Throttled",
"slow down",
)))
},
0_u32,
)
.wait_strategy_fn(|_state, _attempt| WaitDecision::complete())
.name("cond")
.await
.unwrap_err();
let wire = hooks.find("cond", "uf_end").wire.expect("an Err outcome");
assert_eq!(wire.error_type(), Some("Throttled"));
assert_eq!(wire.error_message(), Some("slow down"));
assert_eq!(err.wire(), Some(&wire));
assert_eq!(
recorded_fail(&client),
(Some("Throttled".to_owned()), Some("slow down".to_owned()))
);
}
#[tokio::test]
async fn condition_check_is_a_user_function() {
let client_cell: Arc<std::sync::OnceLock<Arc<InMemoryExecutionClient>>> = Arc::default();
let probe_client = Arc::clone(&client_cell);
let hooks = crate::plugin::recorder::UfHooks::probing(move || {
let succeeded = probe_client.get().is_some_and(|client| {
client
.recorded_updates()
.iter()
.any(|u| u.action() == &OperationAction::Succeed)
});
succeeded.to_string()
});
let (ctx, client) = uf_ctx(&hooks, Vec::new());
client_cell.set(Arc::clone(&client)).expect("set once");
let state = ctx
.wait_for_condition(|_ctx, state: u32| async move { Ok(state + 1) }, 0_u32)
.wait_strategy_fn(|state: u32, _attempt| {
if state >= 1 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
})
.name("cond")
.await
.unwrap();
assert_eq!(state, 1);
let records = hooks.of("cond");
let starts: Vec<_> = records.iter().filter(|r| r.hook == "uf_start").collect();
let ends: Vec<_> = records.iter().filter(|r| r.hook == "uf_end").collect();
assert_eq!((starts.len(), ends.len()), (1, 1), "{records:?}");
let end = ends.first().expect("one end");
assert_eq!((end.info_attempt, end.ok), (Some(1), Some(true)));
assert_eq!(end.probe, "false", "the end hook ran before the SUCCEED");
assert!(
client
.recorded_updates()
.iter()
.any(|u| u.action() == &OperationAction::Succeed)
);
}
#[tokio::test]
async fn check_function_error_fails_immediately() {
let ctx = make_ctx_with_client();
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state: i32, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async {
Err::<i32, BoxError>("check function failed".into())
},
};
let result = exec.execute().await;
assert!(result.is_err());
let err_msg = format!("{:#}", result.unwrap_err());
assert!(err_msg.contains("check function failed"), "got: {err_msg}");
}
#[tokio::test]
async fn replay_frozen_success() {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Succeeded,
result: Some("42".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 3,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let ctx = make_ctx_with_log(vec![(wire_key, record)]);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
#[expect(unreachable_code)] {
panic!("check must NOT execute during replay");
Ok::<i32, BoxError>(0)
}
},
};
let result = exec.execute().await;
assert_eq!(result.unwrap(), 42);
}
#[tokio::test]
async fn replay_frozen_failure() {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Failed,
result: None,
error_type: Some("WaitForConditionError".to_owned()),
error_message: Some("max attempts exceeded".to_owned()),
error_data: None,
stack_trace: None,
attempt: 3,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let ctx = make_ctx_with_log(vec![(wire_key, record)]);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
#[expect(unreachable_code)] {
panic!("check must NOT execute during replay");
Ok::<i32, BoxError>(0)
}
},
};
let result = exec.execute().await;
assert!(result.is_err());
let err_msg = format!("{:#}", result.unwrap_err());
assert!(err_msg.contains("max attempts exceeded"), "got: {err_msg}");
}
#[tokio::test]
async fn replay_exhaustion_keeps_max_checks_exceeded_classification() {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Failed,
result: None,
error_type: Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE.to_owned()),
error_message: Some("max checks exceeded: 3".to_owned()),
error_data: None,
stack_trace: None,
attempt: 2,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let ctx = make_ctx_with_log(vec![(wire_key, record)]);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
#[expect(unreachable_code)] {
panic!("check must NOT execute during replay");
Ok::<i32, BoxError>(0)
}
},
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::WaitForCondition(wfc_err) => match wfc_err.kind() {
WaitForConditionErrorKind::MaxChecksExceeded(details) => {
assert_eq!(
details.checks(),
3,
"replay must derive checks() from the recorded attempt count"
);
}
other => panic!("expected MaxChecksExceeded on replay, got: {other:?}"),
},
other => panic!("expected WaitForCondition error, got: {other:?}"),
}
let wire = err.wire().unwrap();
assert_eq!(
wire.error_type(),
Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE)
);
assert_eq!(wire.error_message(), Some("max checks exceeded: 3"));
}
#[tokio::test]
async fn spawn_executes_on_blessed_task() {
let ctx = make_ctx_with_client();
let handle = ctx
.wait_for_condition(|_ctx, state: i32| async move { Ok(state) }, 5)
.wait_strategy_fn(|_state: i32, _attempt| WaitDecision::complete())
.spawn();
let result = handle.await;
assert_eq!(result.unwrap(), 5);
}
struct UpperSerdes;
impl Serdes<String> for UpperSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(value.to_uppercase())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(wire)
}
}
#[derive(Clone, Copy)]
enum Failure {
Serialize,
Deserialize,
}
struct FailingSerdes {
failure: Failure,
retryable: bool,
}
impl FailingSerdes {
fn error(&self) -> BoxError {
if self.retryable {
crate::serdes::RetryableSerdesError::new("transient").into()
} else {
"plain serdes failure".into()
}
}
}
impl Serdes<String> for FailingSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
match self.failure {
Failure::Serialize => Err(self.error()),
Failure::Deserialize => Ok(value),
}
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
match self.failure {
Failure::Serialize => Ok(wire),
Failure::Deserialize => Err(self.error()),
}
}
}
fn wfc_record(
status: CheckpointStatus,
result: Option<&str>,
error_type: Option<&str>,
attempt: u32,
) -> (String, CheckpointRecord) {
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status,
result: result.map(str::to_owned),
error_type: error_type.map(str::to_owned),
error_message: error_type.map(|_| "recorded".to_owned()),
error_data: None,
stack_trace: None,
attempt,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
(wire_key, record)
}
fn ctx_with(
client: &Arc<InMemoryExecutionClient>,
records: Vec<(String, CheckpointRecord)>,
) -> DurableContext {
DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
)
}
async fn first_check_receives<S, SD>(
records: Vec<(String, CheckpointRecord)>,
initial_state: S,
serdes: SD,
) -> S
where
S: Clone + Send + Sync + std::fmt::Debug + 'static,
SD: Serdes<S>,
{
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = ctx_with(&client, records);
let op_id = ctx.mint_id();
let seen = Arc::new(std::sync::Mutex::new(None::<S>));
let seen_check = Arc::clone(&seen);
let exec = WaitForConditionExecution {
ctx,
op_id,
name: None,
initial_state,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes,
check: move |_ctx, state: S| {
let seen = Arc::clone(&seen_check);
async move {
*seen.lock().unwrap() = Some(state.clone());
Ok::<S, BoxError>(state)
}
},
};
exec.execute().await.expect("the operation succeeds");
seen.lock().unwrap().clone().expect("the check ran")
}
#[tokio::test]
async fn initial_state_round_trip_cases() {
assert_eq!(
first_check_receives(Vec::new(), 5_i32, crate::serdes::JsonSerdes).await,
5
);
let record = wfc_record(CheckpointStatus::Ready, Some("B"), None, 1);
assert_eq!(
first_check_receives(vec![record], "a".to_owned(), UpperSerdes).await,
"B"
);
assert_eq!(
first_check_receives(Vec::new(), "a".to_owned(), UpperSerdes).await,
"A"
);
}
fn flagged_exec(
serdes: FailingSerdes,
called: &Arc<std::sync::atomic::AtomicBool>,
) -> (
DurableContext,
Arc<InMemoryExecutionClient>,
impl Future<Output = Result<String, OperationError>> + Send + 'static,
) {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = ctx_with(&client, Vec::new());
let op_id = ctx.mint_id();
let called = Arc::clone(called);
let exec = WaitForConditionExecution {
ctx: ctx.clone(),
op_id,
name: None,
initial_state: "a".to_owned(),
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes,
check: move |_ctx, state: String| {
called.store(true, std::sync::atomic::Ordering::SeqCst);
async move { Ok::<String, BoxError>(state) }
},
};
(ctx, client, exec.execute())
}
#[tokio::test]
async fn initial_state_round_trip_failure_writes_nothing() {
for failure in [Failure::Serialize, Failure::Deserialize] {
let called = Arc::new(std::sync::atomic::AtomicBool::new(false));
let serdes = FailingSerdes {
failure,
retryable: false,
};
let (_ctx, client, fut) = flagged_exec(serdes, &called);
let err = fut.await.unwrap_err();
match err.kind() {
OperationErrorKind::WaitForCondition(wfc_err) => assert!(
matches!(
wfc_err.kind(),
WaitForConditionErrorKind::SerializationFailed
),
"expected SerializationFailed, got {:?}",
wfc_err.kind()
),
other => panic!("expected WaitForCondition error, got: {other:?}"),
}
assert!(!called.load(std::sync::atomic::Ordering::SeqCst));
assert!(
client.recorded_updates().is_empty(),
"no update is written: {:?}",
client.recorded_updates()
);
}
}
#[tokio::test]
async fn initial_state_retryable_round_trip_failure_ends_invocation() {
for (failure, needle) in [
(Failure::Serialize, "serdes serialize failed"),
(Failure::Deserialize, "serdes deserialize failed"),
] {
let called = Arc::new(std::sync::atomic::AtomicBool::new(false));
let serdes = FailingSerdes {
failure,
retryable: true,
};
let (ctx, client, fut) = flagged_exec(serdes, &called);
crate::driver::test_support::expect_serdes_fault(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
Arc::clone(ctx.coordinator()),
fut,
needle,
)
.await;
assert!(!called.load(std::sync::atomic::Ordering::SeqCst));
assert!(
client.recorded_updates().is_empty(),
"no update is written: {:?}",
client.recorded_updates()
);
}
}
fn wfc_kind(err: &OperationError) -> (&'static str, Option<u32>) {
let OperationErrorKind::WaitForCondition(wfc_err) = err.kind() else {
panic!("expected WaitForCondition error, got: {:?}", err.kind());
};
match wfc_err.kind() {
WaitForConditionErrorKind::MaxChecksExceeded(details) => {
("MaxChecksExceeded", Some(details.checks()))
}
WaitForConditionErrorKind::SerializationFailed => ("SerializationFailed", None),
WaitForConditionErrorKind::CheckFailed => ("CheckFailed", None),
}
}
fn wfc_source(err: &OperationError) -> Option<String> {
let OperationErrorKind::WaitForCondition(wfc_err) = err.kind() else {
panic!("expected WaitForCondition error, got: {:?}", err.kind());
};
std::error::Error::source(wfc_err).map(crate::error::chain_string)
}
#[tokio::test]
async fn check_failure_records_a_propagated_record_unchanged() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = ctx_with(&client, Vec::new());
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
let wire = crate::error::WireError::new(Some("InnerType"), Some("inner"))
.with_error_data(Some("inner-data"))
.with_stack_trace(vec!["inner-frame".to_owned()]);
Err::<i32, BoxError>(Box::new(crate::error::recorded_step_error(wire)))
},
};
drop(exec.execute().await.unwrap_err());
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.action() == &OperationAction::Fail)
.expect("a FAIL update");
let error = fail.error().expect("the FAIL carries an error object");
assert_eq!(error.error_type(), Some("InnerType"));
assert_eq!(error.error_message(), Some("inner"));
assert_eq!(error.error_data(), Some("inner-data"));
assert_eq!(error.stack_trace(), ["inner-frame"]);
}
#[tokio::test]
#[expect(clippy::too_many_lines)] async fn check_failure_records_typed_error_type() {
type MakeErr = fn() -> BoxError;
let cases: [(MakeErr, &str, &str, &str); 4] = [
(
|| "check function failed".into(),
"Error",
"CheckFailed",
"check function failed",
),
(
|| Box::new(crate::TypedError::with_type("Throttled", "slow down")),
"Throttled",
"CheckFailed",
"slow down",
),
(
|| {
Box::new(crate::TypedError::with_type(
"WaitForCondition.MaxChecksExceeded",
"x",
))
},
"WaitForCondition.MaxChecksExceeded",
"MaxChecksExceeded",
"x",
),
(
|| Box::new(crate::TypedError::with_type("SerializationError", "x")),
"SerializationError",
"SerializationFailed",
"x",
),
];
for (make_err, recorded_type, kind, message) in cases {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = ctx_with(&client, Vec::new());
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: move |_ctx, _state: i32| async move { Err::<i32, BoxError>(make_err()) },
};
let live = exec.execute().await.unwrap_err();
let (live_kind, live_checks) = wfc_kind(&live);
assert_eq!(live_kind, kind, "live kind for {recorded_type}");
assert_eq!(
live.wire().and_then(crate::error::WireError::error_type),
Some(recorded_type)
);
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.action() == &OperationAction::Fail)
.expect("a FAIL update");
assert_eq!(
fail.error().and_then(|e| e.error_type()),
Some(recorded_type),
"the FAIL record's type"
);
assert_eq!(
fail.error().and_then(|e| e.error_message()),
Some(message),
"the FAIL record's message"
);
let live_source = wfc_source(&live);
if kind == "MaxChecksExceeded" {
assert_eq!(live_source, None, "MaxChecksExceeded carries no source");
} else {
let source = live_source.expect("the check error is the source");
assert_eq!(source, message, "the flattened source");
}
let record = wfc_record(CheckpointStatus::Failed, None, Some(recorded_type), 0);
let ctx = make_ctx_with_log(vec![record]);
let op_id = ctx.mint_id();
let exec = WaitForConditionExecution::<i32, _, _> {
ctx,
op_id,
name: None,
initial_state: 0,
wait_strategy: Box::new(|_state, _attempt| WaitDecision::complete()),
serdes: crate::serdes::JsonSerdes,
check: |_ctx, _state: i32| async move {
#[expect(unreachable_code)] {
panic!("check must NOT execute during replay");
Ok::<i32, BoxError>(0)
}
},
};
let replayed = exec.execute().await.unwrap_err();
let (replay_kind, replay_checks) = wfc_kind(&replayed);
assert_eq!(replay_kind, kind, "replay kind for {recorded_type}");
assert_eq!(live_checks, replay_checks, "checks() for {recorded_type}");
assert_eq!(
replayed
.wire()
.and_then(crate::error::WireError::error_type),
Some(recorded_type)
);
}
}
#[test]
fn classify_wfc_failure_maps_each_type() {
assert_eq!(
classify_wfc_failure(Some(crate::error::WFC_EXHAUSTED_ERROR_TYPE)),
WfcFailureClass::MaxChecksExceeded
);
assert_eq!(
classify_wfc_failure(Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE)),
WfcFailureClass::SerializationFailed
);
assert_eq!(
classify_wfc_failure(Some("Error")),
WfcFailureClass::CheckFailed
);
assert_eq!(classify_wfc_failure(None), WfcFailureClass::CheckFailed);
}
}
#[cfg(test)]
mod coordinator_tests {
use crate::BoxError;
use crate::builders::wait_for_condition::WaitDecision;
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{
ScriptedClient, drive_handler, named, operation_like, scripted_ctx, wire_err,
};
use aws_sdk_lambda::types::{OperationAction, OperationStatus, StepDetails};
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Duration;
#[tokio::test(start_paused = true)]
async fn wfc_resumed_in_invocation_fires_start() {
use crate::driver::test_support::{RefreshClient, poll_once};
use crate::plugin::AttemptOutcome::Succeeded;
let client = Arc::new(RefreshClient::new(|req| {
let record =
|status: OperationStatus, update: &aws_sdk_lambda::types::OperationUpdate| {
let mut details = StepDetails::builder().attempt(1);
if let Some(payload) = update.payload() {
details = details.result(payload);
}
operation_like(update, status)
.step_details(details.build())
.build()
.expect("all required fields set")
};
let mut ops = Vec::new();
if let Some(retry) = named(req.updates, "cond", &OperationAction::Retry) {
ops.push(record(OperationStatus::Pending, retry));
}
if req.empty_number == 1
&& let Some(retry) = named(req.history, "cond", &OperationAction::Retry)
{
ops.push(record(OperationStatus::Ready, retry));
}
Ok(ops)
}));
let hooks = crate::plugin::test_support::OpHooks::default();
let ctx = crate::DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
None,
0,
Some(hooks.plugins()),
);
let outcome = drive_handler(ctx, None, |ctx| async move {
let mut cond = std::pin::pin!(
ctx.wait_for_condition(|_ctx, state: u32| async move { Ok(state + 1) }, 0_u32)
.wait_strategy_fn(|state: u32, _attempt| {
if state >= 2 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
})
.name("cond")
.spawn()
);
poll_once(cond.as_mut()).await;
ctx.step(|_| async {
tokio::time::sleep(Duration::from_secs(3)).await;
Ok(1_u32)
})
.name("s")
.await
.map_err(wire_err)?;
let state = cond.await.map_err(wire_err)?;
Ok(state.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("2".to_owned()));
let records = hooks.of("cond");
let shapes: Vec<_> = records
.iter()
.map(crate::plugin::test_support::OpRecord::shape)
.collect();
assert_eq!(
shapes,
[
("start", false, None, None),
("attempt_start", false, Some(1), None),
("attempt_end", false, Some(1), Some(Succeeded)),
("start", true, None, None),
("attempt_start", false, Some(2), None),
("attempt_end", false, Some(2), Some(Succeeded)),
("end", false, None, None),
]
);
let starts = records.iter().filter(|r| r.hook.ends_with("start"));
let info_attempts: Vec<_> = starts.map(|r| r.info_attempt).collect();
assert_eq!(info_attempts, [None, Some(1), Some(1), Some(2)]);
let results: Vec<_> = records.iter().map(|r| r.result.as_deref()).collect();
assert_eq!(results.get(5..), Some(&[Some("2"), Some("2")][..]));
let statuses: Vec<_> = records.iter().map(|r| r.status).collect();
assert_eq!(
statuses,
[
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Pending),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Started),
Some(crate::OperationStatus::Succeeded),
Some(crate::OperationStatus::Succeeded),
]
);
}
#[tokio::test(start_paused = true)]
async fn condition_check_resumes_on_sibling_response() {
let (events, _guard) = crate::driver::test_support::EventMessages::install();
let client = Arc::new(ScriptedClient::new(|updates, history| {
if named(updates, "s", &OperationAction::Succeed).is_none() {
return Vec::new();
}
named(history, "cond", &OperationAction::Start)
.map(|start| {
operation_like(start, OperationStatus::Ready)
.step_details(StepDetails::builder().attempt(1).result("1").build())
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
}));
let ctx = scripted_ctx(&client);
let checks = Arc::new(AtomicU32::new(0));
let check_count = Arc::clone(&checks);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cond = ctx
.wait_for_condition(
move |_ctx, state: i32| {
let checks = Arc::clone(&check_count);
async move {
checks.fetch_add(1, Ordering::SeqCst);
Ok::<i32, BoxError>(state + 1)
}
},
0,
)
.wait_strategy_fn(|state: i32, _attempt| {
if state >= 2 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(Duration::from_secs(1))
}
})
.name("cond")
.spawn();
ctx.step(move |_| async move {
for _ in 0..1000 {
if body_client.received("cond", &OperationAction::Retry) {
return Ok(1_u32);
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
Err("the condition never retried".into())
})
.name("s")
.await
.map_err(wire_err)?;
let state = cond.await.map_err(wire_err)?;
Ok(state.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("2".to_owned()));
assert_eq!(checks.load(Ordering::SeqCst), 2, "the check ran twice");
assert_eq!(
events.replayed(),
0,
"a resumed operation emits no replay event"
);
}
#[tokio::test(start_paused = true)]
async fn resumed_terminal_record_returns_without_replay_event() {
let (events, _guard) = crate::driver::test_support::EventMessages::install();
let client = Arc::new(ScriptedClient::new(|updates, history| {
if named(updates, "s", &OperationAction::Succeed).is_none() {
return Vec::new();
}
named(history, "cond", &OperationAction::Start)
.map(|start| {
operation_like(start, OperationStatus::Succeeded)
.step_details(StepDetails::builder().attempt(1).result("5").build())
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
}));
let ctx = scripted_ctx(&client);
let checks = Arc::new(AtomicU32::new(0));
let check_count = Arc::clone(&checks);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cond = ctx
.wait_for_condition(
move |_ctx, state: i32| {
let checks = Arc::clone(&check_count);
async move {
checks.fetch_add(1, Ordering::SeqCst);
Ok::<i32, BoxError>(state + 1)
}
},
0,
)
.wait_strategy_fn(|_state: i32, _attempt| {
WaitDecision::continue_with(Duration::from_secs(1))
})
.name("cond")
.spawn();
ctx.step(move |_| async move {
for _ in 0..1000 {
if body_client.received("cond", &OperationAction::Retry) {
return Ok(1_u32);
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
Err("the condition never retried".into())
})
.name("s")
.await
.map_err(wire_err)?;
let state = cond.await.map_err(wire_err)?;
Ok(state.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("5".to_owned()));
assert_eq!(checks.load(Ordering::SeqCst), 1, "the check ran once");
assert_eq!(
events.replayed(),
0,
"a resumed record emits no replay event"
);
}
}
#[cfg(test)]
mod start_write_tests {
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{LatencyClient, drive_handler, latency_ctx, wire_err};
use aws_sdk_lambda::types::OperationAction;
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[tokio::test(start_paused = true)]
async fn condition_check_starts_before_start_response() {
let client = Arc::new(LatencyClient::new(Duration::from_millis(500)));
let ctx = latency_ctx(&client, None);
let started = Arc::new(Mutex::new(None));
let check_started = Arc::clone(&started);
let clock = Arc::clone(&client);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let state = ctx
.wait_for_condition(
move |_, state: u32| {
let started = Arc::clone(&check_started);
let clock = Arc::clone(&clock);
async move {
*started.lock().expect("lock") = Some(clock.elapsed());
Ok(state + 1)
}
},
0,
)
.wait_strategy_fn(|_state, _attempt| {
crate::builders::wait_for_condition::WaitDecision::complete()
})
.name("c")
.await
.map_err(wire_err)?;
Ok(state.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("1".to_owned()));
assert_eq!(
*started.lock().expect("lock"),
Some(Duration::ZERO),
"the check starts before START is answered"
);
let start = client.request_with(&OperationAction::Start).expect("START");
let succeed = client
.request_with(&OperationAction::Succeed)
.expect("SUCCEED");
assert_eq!(start.received, Duration::ZERO);
assert_eq!(
succeed.received,
Duration::from_millis(500),
"SUCCEED waits for START"
);
}
}