use std::future::Future;
use std::time::Duration;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use crate::BoxError;
use crate::builders::Submitter;
use crate::checkpoint_coalescer::ScopeClose;
use crate::context::{DurableContext, StepContext, SubmittedWrite, run_body};
use crate::engine::{CheckpointStatus, OperationId};
use crate::error::{CallbackError, CallbackErrorKind, OperationError, OperationErrorKind};
use crate::future::Callback;
use crate::plugin::UserBody;
use crate::serdes::{PassthroughSerdes, PayloadOrigin, Serdes, SerdesContext};
use crate::start::StartStep;
use tracing::Instrument as _;
pub(crate) const CALLBACK_SUB_TYPE: &str = "Callback";
pub(crate) const WFCB_SUB_TYPE: &str = "WaitForCallback";
pub(crate) struct CreateCallbackExecution<O, S> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) timeout: Option<Duration>,
pub(crate) heartbeat: Option<Duration>,
pub(crate) sub_type: crate::OperationSubType,
pub(crate) plugin_name: Option<String>,
pub(crate) serdes: S,
pub(crate) _marker: std::marker::PhantomData<O>,
}
impl<O, S> CreateCallbackExecution<O, S>
where
O: Send + 'static,
S: Serdes<O>,
{
pub(crate) async fn execute(self) -> Result<Callback<O>, OperationError> {
let started = self.start();
self.execute_after_start(started).await
}
pub(crate) fn execute_started(
self,
) -> impl Future<Output = Result<Callback<O>, OperationError>> + Send + 'static {
let started = self.start();
self.execute_after_start(started)
}
fn start(&self) -> Result<StartStep, OperationError> {
let positional_id = self.op_id.positional();
let wire_id = self.op_id.wire();
crate::start::start_step(
&self.ctx,
&crate::start::StartSpec {
op_id: &self.op_id,
operation_type: "Callback",
sub_type: self.sub_type.as_str(),
name: self.name.as_deref(),
order: crate::start::StartOrder::AwaitUrgentResponse,
},
|view| {
let in_flight = matches!(
view.status,
CheckpointStatus::Started | CheckpointStatus::Pending
);
if in_flight && self.recorded_callback_id(positional_id).is_some() {
crate::start::Cycle::Resume
} else {
crate::start::Cycle::Record
}
},
|| self.start_update(wire_id),
|plugins, _| {
let info = self.info(wire_id).with_replay(true);
crate::plugin::operation_started(plugins, &info);
},
)
}
async fn execute_after_start(
self,
started: Result<StartStep, OperationError>,
) -> Result<Callback<O>, OperationError> {
let started = started?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let start = match started {
StartStep::Refused => return std::future::pending().await,
StartStep::Queued(write) => write,
StartStep::Recorded(view) => {
let callback_id = self
.recorded_callback_id(&positional_id)
.ok_or_else(|| missing_callback_id_error(&wire_id, view.status.wire_str()))?;
if matches!(
view.status,
CheckpointStatus::Started | CheckpointStatus::Pending
) {
return Ok(self.into_pending(callback_id));
}
if let Some(plugins) = self.ctx.plugins()
&& is_terminal_callback_status(&view.status)
{
let info = self
.info(&wire_id)
.with_replay(!plugins.is_updated(&wire_id));
crate::plugin::operation_ended(plugins, &info);
}
let outcome = callback_outcome(
&self.ctx,
&positional_id,
self.name.as_deref(),
self.sub_type.as_str(),
&self.serdes,
&view,
true,
)
.await?;
return Ok(Callback::new_settled(callback_id, outcome));
}
};
if let Err(err) = start.await {
return self.ctx.checkpoint_failure_unrecoverable(err).await;
}
let callback_id = self.recorded_callback_id(&positional_id).ok_or_else(|| {
missing_callback_id_error(&wire_id, CheckpointStatus::Started.wire_str())
})?;
if let Some(plugins) = self.ctx.plugins() {
let info = self
.info(&wire_id)
.with_status(crate::OperationStatus::Started)
.with_replay(false);
crate::plugin::operation_started(plugins, &info);
}
Ok(self.into_pending(callback_id))
}
fn recorded_callback_id(&self, positional_id: &str) -> Option<String> {
self.ctx
.checkpoint_callback_id(positional_id)
.filter(|id| !id.is_empty())
}
fn start_update(&self, wire_id: &str) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Callback)
.sub_type(self.sub_type.as_str())
.action(OperationAction::Start);
if let Some(n) = &self.name {
builder = builder.name(n.clone());
}
if let Some(parent_wire) = self.ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
let cb_opts = build_callback_options(self.timeout, self.heartbeat);
if let Some(opts) = cb_opts {
builder = builder.callback_options(opts);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn info(&self, wire_id: &str) -> crate::plugin::OperationInfo {
callback_info(
&self.ctx,
wire_id,
self.sub_type.as_str(),
self.name.as_deref(),
self.plugin_name.as_deref(),
)
}
fn into_pending(self, callback_id: String) -> Callback<O> {
let Self {
ctx,
op_id,
name,
sub_type,
plugin_name,
serdes,
..
} = self;
let identity = crate::future::CallbackIdentity {
name: name.clone(),
plugin_name,
sub_type: sub_type.as_str().to_owned(),
};
let wire_id = op_id.wire().to_owned();
let positional_id = op_id.positional().to_owned();
let resolver_ctx = ctx.clone();
let resolver: crate::future::CallbackResolver<O> = Box::new(move || {
Box::pin(async move {
#[expect(clippy::expect_used)]
let view = resolver_ctx
.checkpoint_status_view(&positional_id)
.expect("a settled callback record is still in the log");
callback_outcome(
&resolver_ctx,
&positional_id,
name.as_deref(),
sub_type.as_str(),
&serdes,
&view,
false,
)
.await
.and_then(|outcome| outcome)
})
});
Callback::new_pending(callback_id, ctx, wire_id, identity, resolver)
}
}
pub(crate) fn is_terminal_callback_status(status: &CheckpointStatus) -> bool {
matches!(
status,
CheckpointStatus::Succeeded
| CheckpointStatus::Failed
| CheckpointStatus::TimedOut
| CheckpointStatus::Stopped
| CheckpointStatus::Cancelled
)
}
pub(crate) fn callback_info(
ctx: &DurableContext,
wire_id: &str,
sub_type: &str,
name: Option<&str>,
plugin_name: Option<&str>,
) -> crate::plugin::OperationInfo {
let info = crate::plugin::OperationInfo::for_operation(
ctx,
wire_id,
crate::OperationKind::Callback,
sub_type,
name,
);
let info = match info.status() {
Some(
crate::OperationStatus::Failed
| crate::OperationStatus::Cancelled
| crate::OperationStatus::Stopped,
) => {
let error = info.error().cloned().map(normalized_failure_wire);
info.with_error(error)
}
Some(crate::OperationStatus::TimedOut) => {
let error = info.error().cloned().map(normalized_timeout_wire);
info.with_error(error)
}
_ => info,
};
match plugin_name {
Some(plugin_name) => info.with_name(Some(plugin_name.to_owned())),
None => info,
}
}
async fn callback_outcome<O, S: Serdes<O>>(
ctx: &DurableContext,
positional_id: &str,
name: Option<&str>,
sub_type: &str,
serdes: &S,
view: &crate::engine::CheckpointStatusView,
emit_replayed: bool,
) -> Result<Result<O, OperationError>, OperationError>
where
O: Send + 'static,
{
let wire_id = crate::engine::compute_wire_id_public(positional_id);
let emit = || {
if emit_replayed {
ctx.emit_operation_replayed(&wire_id, name, "Callback", Some(sub_type), view.attempt);
}
};
match view.status {
CheckpointStatus::Succeeded => {
let payload = ctx.checkpoint_result_payload(positional_id);
let result_str = payload.unwrap_or_else(|| "null".to_owned());
let serdes_ctx = SerdesContext::new(&wire_id, ctx.execution_arn());
match deserialize_callback_result(ctx, name, serdes, result_str, serdes_ctx).await {
Ok(value) => {
emit();
Ok(Ok(value))
}
Err(error) => Ok(Err(error)),
}
}
CheckpointStatus::Failed | CheckpointStatus::Cancelled | CheckpointStatus::Stopped => {
emit();
let wire = ctx.checkpoint_wire_error(positional_id).unwrap_or_default();
Ok(Err(replayed_callback_error(
wire,
&wire_id,
view.status.wire_str(),
)))
}
CheckpointStatus::TimedOut => {
emit();
let wire = ctx.checkpoint_wire_error(positional_id);
Ok(Err(timed_out_callback_error(
wire,
&wire_id,
view.status.wire_str(),
)))
}
CheckpointStatus::Unknown(ref raw) => {
Err(ctx.unrecognized_status_error(&wire_id, raw))
}
CheckpointStatus::Started | CheckpointStatus::Pending | CheckpointStatus::Ready => {
Err(callback_internal_error(&format!(
"unexpected checkpointed status: {:?}",
view.status
)))
}
}
}
fn timed_out_callback_error(
wire: Option<crate::error::WireError>,
wire_id: &str,
status: &str,
) -> OperationError {
let wire = normalized_timeout_wire(wire.unwrap_or_default());
let kind = if wire.error_type() == Some("Callback.Heartbeat") {
CallbackErrorKind::HeartbeatTimedOut
} else {
CallbackErrorKind::TimedOut
};
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(kind, None)))
.with_operation(wire_id, status)
.with_wire(wire)
}
pub(crate) struct WaitForCallbackExecution<O, F, S> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) timeout: Option<Duration>,
pub(crate) heartbeat: Option<Duration>,
pub(crate) submitter: F,
pub(crate) submitter_retry: Option<crate::RetryStrategy>,
pub(crate) serdes: S,
pub(crate) _marker: std::marker::PhantomData<O>,
}
impl<O, F, S> WaitForCallbackExecution<O, F, S>
where
O: Send + 'static,
F: Submitter,
S: Serdes<O>,
{
#[cfg(test)]
pub(crate) async fn execute(self) -> Result<O, OperationError> {
let (core, rest, serdes) = self.split();
let started = core.start();
Self::run(core, rest, serdes, started).await
}
pub(crate) fn execute_started(
self,
) -> impl Future<Output = Result<O, OperationError>> + Send + 'static {
let (core, rest, serdes) = self.split();
let started = core.start();
Self::run(core, rest, serdes, started)
}
fn split(self) -> (WfcbCore, WfcbBody<F>, S) {
let Self {
ctx,
op_id,
name,
timeout,
heartbeat,
submitter,
submitter_retry,
serdes,
..
} = self;
let plugin_names = InnerPluginNames::of(name.as_deref());
let core = WfcbCore { ctx, op_id, name };
let rest = WfcbBody {
plugin_names,
timeout,
heartbeat,
submitter,
submitter_retry,
};
(core, rest, serdes)
}
async fn run(
core: WfcbCore,
rest: WfcbBody<F>,
serdes: S,
started: Result<StartStep, OperationError>,
) -> Result<O, OperationError> {
let ctx = core.ctx.clone();
let name = core.name.clone();
let wire_id = core.op_id.wire().to_owned();
let settled = Self::settle_raw(core, rest, started).await?;
let serdes_ctx = SerdesContext::new(&wire_id, ctx.execution_arn());
let value = deserialize_callback_result(
&ctx,
name.as_deref(),
&serdes,
settled.payload,
serdes_ctx,
)
.await?;
if let Some(attempt) = settled.replayed_attempt {
ctx.emit_operation_replayed(
&wire_id,
name.as_deref(),
"Context",
Some(WFCB_SUB_TYPE),
attempt,
);
}
Ok(value)
}
async fn settle_raw(
core: WfcbCore,
rest: WfcbBody<F>,
started: Result<StartStep, OperationError>,
) -> Result<WfcbRaw, OperationError> {
let WfcbBody {
plugin_names,
timeout,
heartbeat,
submitter,
submitter_retry,
} = rest;
let core_admission = std::sync::Arc::clone(core.ctx.admission());
let core_wire_id = core.op_id.wire().to_owned();
match core.before(started?).await? {
WfcbPrelude::Failed(err) => Err(err),
WfcbPrelude::Replayed { payload, attempt } => Ok(WfcbRaw {
payload,
replayed_attempt: Some(attempt),
}),
WfcbPrelude::Reconstruct { child_ctx } => {
let child_span = child_ctx.replay_span();
let active = UserBody::Context(child_ctx.prefix().to_owned());
let scope = ScopeClose::new(&core_admission, &core_wire_id);
let payload = run_body(
active.into(),
run_wfcb_body(
child_ctx,
plugin_names,
timeout,
heartbeat,
submitter,
submitter_retry,
),
)
.instrument(child_span)
.await;
drop(scope);
payload.map(|payload| WfcbRaw {
payload,
replayed_attempt: None,
})
}
WfcbPrelude::Run {
child_ctx,
start,
info,
after,
} => {
let child_span = child_ctx.replay_span();
let active = UserBody::Context(child_ctx.prefix().to_owned());
let scope = ScopeClose::new(after.ctx.admission(), &after.wire_id);
let body_result = crate::plugin::run_user_function(
after.ctx.shared_plugins(),
info,
active,
child_span,
move || async move {
let result = run_wfcb_body(
child_ctx,
plugin_names,
timeout,
heartbeat,
submitter,
submitter_retry,
)
.await;
drop(scope);
result
},
extract_wire_error,
)
.await;
after
.settle(start, body_result)
.await
.map(|payload| WfcbRaw {
payload,
replayed_attempt: None,
})
}
}
}
}
struct WfcbRaw {
payload: String,
replayed_attempt: Option<u32>,
}
struct WfcbBody<F> {
plugin_names: InnerPluginNames,
timeout: Option<Duration>,
heartbeat: Option<Duration>,
submitter: F,
submitter_retry: Option<crate::RetryStrategy>,
}
struct InnerPluginNames {
callback: Option<String>,
submitter: Option<String>,
}
impl InnerPluginNames {
fn of(name: Option<&str>) -> Self {
Self {
callback: name.map(|n| format!("{n}-callback")),
submitter: name.map(|n| format!("{n}-submitter")),
}
}
}
fn wfcb_info(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
) -> crate::plugin::OperationInfo {
crate::plugin::OperationInfo::for_operation(
ctx,
wire_id,
crate::OperationKind::Context,
WFCB_SUB_TYPE,
name,
)
}
struct WfcbCore {
ctx: DurableContext,
op_id: OperationId,
name: Option<String>,
}
enum WfcbPrelude {
Failed(OperationError),
Replayed { payload: String, attempt: u32 },
Reconstruct {
child_ctx: DurableContext,
},
Run {
child_ctx: DurableContext,
start: Option<SubmittedWrite>,
info: std::sync::Arc<crate::plugin::OperationInfo>,
after: WfcbAfter,
},
}
struct WfcbAfter {
ctx: DurableContext,
wire_id: String,
name: Option<String>,
}
impl WfcbCore {
fn start(&self) -> Result<StartStep, OperationError> {
let wire_id = self.op_id.wire();
crate::start::start_step(
&self.ctx,
&crate::start::StartSpec {
op_id: &self.op_id,
operation_type: "Context",
sub_type: WFCB_SUB_TYPE,
name: self.name.as_deref(),
order: crate::start::StartOrder::QueueThenHook,
},
|view| match view.status {
CheckpointStatus::Succeeded | CheckpointStatus::Failed => {
crate::start::Cycle::Record
}
_ => crate::start::Cycle::Resume,
},
|| {
build_wfcb_update(
wire_id,
self.name.as_deref(),
OperationAction::Start,
&self.ctx,
)
},
|plugins, is_replay| {
let info = wfcb_info(&self.ctx, wire_id, self.name.as_deref());
let info = if is_replay {
info.with_replay(true)
} else {
info.with_status(crate::OperationStatus::Started)
.with_replay(false)
};
crate::plugin::operation_started(plugins, &info);
},
)
}
async fn before(self, started: StartStep) -> Result<WfcbPrelude, OperationError> {
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let Self { ctx, name, .. } = self;
let start = match started {
StartStep::Refused => return std::future::pending().await,
StartStep::Queued(write) => Some(write),
StartStep::Recorded(view) => match view.status {
CheckpointStatus::Succeeded if view.replay_children => {
let child_ctx = ctx.new_child(&positional_id, name.as_deref());
return Ok(WfcbPrelude::Reconstruct { child_ctx });
}
CheckpointStatus::Succeeded => {
let payload = ctx
.checkpoint_result_payload(&positional_id)
.unwrap_or_else(|| "null".to_owned());
return Ok(WfcbPrelude::Replayed {
payload,
attempt: view.attempt,
});
}
CheckpointStatus::Failed => {
ctx.emit_operation_replayed(
&wire_id,
name.as_deref(),
"Context",
Some(WFCB_SUB_TYPE),
view.attempt,
);
let wire = ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Ok(WfcbPrelude::Failed(wfcb_failed_error(
wire,
&wire_id,
view.status.wire_str(),
)));
}
_ => None,
},
};
let child_ctx = ctx.new_child(&positional_id, name.as_deref());
let info = wfcb_info(&ctx, &wire_id, name.as_deref());
let info = if start.is_some() {
info.with_status(crate::OperationStatus::Started)
} else {
info
};
let info = std::sync::Arc::new(info.with_replay(child_ctx.is_replaying()));
Ok(WfcbPrelude::Run {
child_ctx,
start,
info,
after: WfcbAfter { ctx, wire_id, name },
})
}
}
impl WfcbAfter {
async fn settle(
self,
start: Option<SubmittedWrite>,
body_result: crate::plugin::Ran<String, OperationError>,
) -> Result<String, OperationError> {
let Self { ctx, wire_id, name } = self;
match body_result.into_parts() {
Ok(raw) => {
let mut builder = OperationUpdate::builder()
.id(wire_id.clone())
.r#type(OperationType::Context)
.sub_type(WFCB_SUB_TYPE.to_owned())
.action(OperationAction::Succeed)
.payload(raw.clone());
if let Some(n) = &name {
builder = builder.name(n.clone());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
if ctx.preserves_children(1) {
builder = builder.context_options(replay_children_options());
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
ctx.await_retained_start(&wire_id, start, OperationAction::Succeed)
.await?;
if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = ctx.plugins() {
let info = wfcb_info(&ctx, &wire_id, name.as_deref())
.with_status(crate::OperationStatus::Succeeded)
.with_result(Some(raw.clone()))
.with_replay(false);
crate::plugin::operation_ended(plugins, &info);
}
Ok(raw)
}
Err((err, _))
if matches!(
err.kind(),
OperationErrorKind::NonDeterministicExecution(error)
if matches!(
error.kind(),
crate::error::NonDeterministicExecutionErrorKind::UnrecognizedStatus(_)
)
) =>
{
Err(err)
}
Err((err, wire)) => {
let mut builder = OperationUpdate::builder()
.id(wire_id.clone())
.r#type(OperationType::Context)
.sub_type(WFCB_SUB_TYPE.to_owned())
.action(OperationAction::Fail);
if let Some(n) = &name {
builder = builder.name(n.clone());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
builder = builder.error(wire.to_error_object());
if ctx.preserves_children(1) {
builder = builder.context_options(replay_children_options());
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
ctx.await_retained_start(&wire_id, start, OperationAction::Fail)
.await?;
if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(client_err).await;
}
if let Some(plugins) = ctx.plugins() {
let info = wfcb_info(&ctx, &wire_id, name.as_deref())
.with_status(crate::OperationStatus::Failed)
.with_error(Some(wire.clone()))
.with_replay(false);
crate::plugin::operation_ended(plugins, &info);
}
Err(wfcb_live_error(err, wire, &wire_id))
}
}
}
}
async fn run_wfcb_body<F>(
child_ctx: DurableContext,
plugin_names: InnerPluginNames,
timeout: Option<Duration>,
heartbeat: Option<Duration>,
submitter: F,
submitter_retry: Option<crate::RetryStrategy>,
) -> Result<String, OperationError>
where
F: Submitter,
{
let cb_exec: CreateCallbackExecution<String, PassthroughSerdes> = CreateCallbackExecution {
ctx: child_ctx.clone(),
op_id: child_ctx.mint_id(),
name: None,
timeout,
heartbeat,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: plugin_names.callback,
serdes: PassthroughSerdes,
_marker: std::marker::PhantomData,
};
let cb = cb_exec.execute().await?;
let callback_id = cb.id().to_owned();
let step_exec = crate::step::StepExecution {
ctx: child_ctx.clone(),
op_id: child_ctx.mint_id(),
name: None,
retry_strategy: submitter_retry,
serdes: crate::serdes::JsonSerdes,
semantics: crate::step::StepSemantics::default(),
sub_type: crate::OperationSubType::STEP,
plugin_name: plugin_names.submitter,
closure: move |step_ctx: StepContext| async move {
(submitter)(step_ctx, callback_id).await?;
Ok(())
},
_marker: std::marker::PhantomData,
};
if let Err(e) = step_exec.execute().await {
if matches!(e.kind(), OperationErrorKind::Step(_)) {
return Err(OperationError::from_kind(OperationErrorKind::Callback(
CallbackError::new(CallbackErrorKind::SubmitterFailed, Some(Box::new(e))),
)));
}
return Err(e);
}
cb.result().await
}
fn build_callback_options(
timeout: Option<Duration>,
heartbeat: Option<Duration>,
) -> Option<aws_sdk_lambda::types::CallbackOptions> {
#[expect(clippy::cast_possible_truncation)] let timeout_secs = timeout.map_or(0, |d| {
(d.as_secs_f64().ceil() as i64).min(i64::from(i32::MAX)) as i32
});
#[expect(clippy::cast_possible_truncation)] let heartbeat_secs = heartbeat.map_or(0, |d| {
(d.as_secs_f64().ceil() as i64).min(i64::from(i32::MAX)) as i32
});
if timeout_secs == 0 && heartbeat_secs == 0 {
return None;
}
Some(
aws_sdk_lambda::types::CallbackOptions::builder()
.timeout_seconds(timeout_secs)
.heartbeat_timeout_seconds(heartbeat_secs)
.build(),
)
}
fn build_wfcb_update(
wire_id: &str,
name: Option<&str>,
action: OperationAction,
ctx: &DurableContext,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Context)
.sub_type(WFCB_SUB_TYPE.to_owned())
.action(action);
if let Some(n) = name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn replay_children_options() -> aws_sdk_lambda::types::ContextOptions {
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build()
}
const WFCB_EXTERNAL_FAILURE: &str = "Callback.ExternalFailure";
const CALLBACK_TIMEOUT: &str = "Callback.Timeout";
const CALLBACK_HEARTBEAT: &str = "Callback.Heartbeat";
const CALLBACK_TIMED_OUT_MESSAGE: &str = "Callback timed out";
fn timeout_message(recorded: Option<&str>) -> &str {
match recorded {
Some(message) if !message.is_empty() => message,
_ => CALLBACK_TIMED_OUT_MESSAGE,
}
}
fn normalized_timeout_wire(wire: crate::error::WireError) -> crate::error::WireError {
let message = timeout_message(wire.error_message()).to_owned();
wire.with_error_message(Some(message))
}
const CALLBACK_FAILED_MESSAGE: &str = "Callback failed";
fn callback_message(recorded: Option<&str>) -> &str {
match recorded {
Some(message) if !message.is_empty() => message,
_ => CALLBACK_FAILED_MESSAGE,
}
}
fn normalized_failure_wire(wire: crate::error::WireError) -> crate::error::WireError {
let message = callback_message(wire.error_message()).to_owned();
wire.with_error_message(Some(message))
}
const WFCB_SUBMITTER_FAILED: &str = "Callback.SubmitterFailed";
const WFCB_INTERNAL: &str = "Callback.Internal";
const WFCB_DESERIALIZATION_FAILED: &str = "Callback.DeserializationFailed";
fn classify_wfcb_outer_error(error_type: Option<&str>) -> CallbackErrorKind {
match error_type {
Some(CALLBACK_TIMEOUT) => CallbackErrorKind::TimedOut,
Some(CALLBACK_HEARTBEAT) => CallbackErrorKind::HeartbeatTimedOut,
Some(WFCB_SUBMITTER_FAILED) => CallbackErrorKind::SubmitterFailed,
Some(WFCB_INTERNAL) => CallbackErrorKind::Internal,
Some(WFCB_DESERIALIZATION_FAILED) => CallbackErrorKind::DeserializationFailed,
_ => CallbackErrorKind::ExternalFailure,
}
}
fn replayed_callback_error_with_kind(
wire: crate::error::WireError,
wire_id: &str,
status: &str,
kind: CallbackErrorKind,
) -> OperationError {
let source = match kind {
CallbackErrorKind::TimedOut | CallbackErrorKind::HeartbeatTimedOut => None,
_ => Some(crate::error::ReplayedFailure::source_from(wire.clone())),
};
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind, source,
)))
.with_operation(wire_id, status)
.with_wire(wire)
}
fn replayed_callback_error(
wire: crate::error::WireError,
wire_id: &str,
status: &str,
) -> OperationError {
replayed_callback_error_with_kind(
normalized_failure_wire(wire),
wire_id,
status,
CallbackErrorKind::ExternalFailure,
)
}
fn wfcb_failed_error(wire: crate::error::WireError, wire_id: &str, status: &str) -> OperationError {
let kind = classify_wfcb_outer_error(wire.error_type());
replayed_callback_error_with_kind(wire, wire_id, status, kind)
}
fn wfcb_live_error(
err: OperationError,
wire: crate::error::WireError,
wire_id: &str,
) -> OperationError {
let status = CheckpointStatus::Failed.wire_str();
match err.kind() {
OperationErrorKind::Callback(cb_err)
if matches!(cb_err.kind(), CallbackErrorKind::ExternalFailure) =>
{
wfcb_failed_error(wire, wire_id, status)
}
OperationErrorKind::Callback(_) => err.with_operation(wire_id, status).with_wire(wire),
_ => OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::Internal,
Some(Box::new(err)),
)))
.with_operation(wire_id, status)
.with_wire(wire),
}
}
fn extract_wire_error(err: &OperationError) -> crate::error::WireError {
let OperationErrorKind::Callback(cb_err) = err.kind() else {
return crate::error::wire_error_with_type(err, WFCB_INTERNAL);
};
match cb_err.kind() {
CallbackErrorKind::TimedOut => timeout_wire(err, CALLBACK_TIMEOUT),
CallbackErrorKind::HeartbeatTimedOut => timeout_wire(err, CALLBACK_HEARTBEAT),
CallbackErrorKind::ExternalFailure => {
let external = err.wire();
crate::error::WireError::new(
Some(WFCB_EXTERNAL_FAILURE),
Some(callback_message(
external.and_then(crate::error::WireError::error_message),
)),
)
.with_error_data(external.and_then(crate::error::WireError::error_data))
}
CallbackErrorKind::SubmitterFailed => {
crate::error::wire_error_with_type(cb_err, WFCB_SUBMITTER_FAILED)
}
CallbackErrorKind::Internal => crate::error::wire_error_with_type(cb_err, WFCB_INTERNAL),
CallbackErrorKind::DeserializationFailed => {
crate::error::wire_error_with_type(cb_err, WFCB_DESERIALIZATION_FAILED)
}
}
}
fn timeout_wire(err: &OperationError, error_type: &str) -> crate::error::WireError {
let recorded = err.wire();
crate::error::WireError::new(
Some(error_type),
Some(timeout_message(
recorded.and_then(crate::error::WireError::error_message),
)),
)
.with_error_data(recorded.and_then(crate::error::WireError::error_data))
}
fn callback_deser_error(boundary: &str, e: BoxError) -> OperationError {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::DeserializationFailed,
Some(crate::error::ContextualError::source_from(boundary, e)),
)))
}
pub(crate) async fn deserialize_callback_result<O, S: Serdes<O>>(
ctx: &DurableContext,
name: Option<&str>,
serdes: &S,
payload: String,
serdes_ctx: SerdesContext,
) -> Result<O, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx
.hold_activity(serdes.deserialize(payload, serdes_ctx.with_origin(PayloadOrigin::External)))
.await
{
Ok(value) => Ok(value),
Err(err) => {
let e = ctx
.route_serdes_error("deserialize", &operation_id, name, err)
.await;
Err(callback_deser_error("callback serdes", e))
}
}
}
fn callback_internal_error(msg: &str) -> OperationError {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::Internal,
Some(msg.to_owned().into()),
)))
}
fn missing_callback_id_error(wire_id: &str, status: &str) -> OperationError {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::Internal,
Some(format!("no callback ID found for started callback: {wire_id}").into()),
)))
.with_operation(wire_id, status)
}
#[cfg(test)]
#[expect(clippy::disallowed_methods)]
mod tests {
use super::*;
use crate::client::InMemoryExecutionClient;
use crate::engine::{CheckpointLog, CheckpointRecord};
use crate::error::{ChildContextError, ChildContextErrorKind};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::sync::Arc;
fn ctx_with_log(records: Vec<(String, CheckpointRecord)>) -> DurableContext {
let log = CheckpointLog::from_records(records);
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:123:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
client,
"token-1".to_owned(),
)
}
fn callback_record(
wire_id: &str,
status: CheckpointStatus,
result: Option<&str>,
error_type: Option<&str>,
error_message: Option<&str>,
callback_id: Option<&str>,
) -> (String, CheckpointRecord) {
(
wire_id.to_owned(),
CheckpointRecord {
id: wire_id.to_owned(),
status,
result: result.map(String::from),
error_type: error_type.map(String::from),
error_message: error_message.map(String::from),
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: callback_id.map(String::from),
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
#[tokio::test]
async fn replay_succeeded() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#""hello""#),
None,
None,
Some("cb-123"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("test".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should succeed");
assert_eq!(cb.id(), "cb-123");
let value = cb.result().await.expect("should have value");
assert_eq!(value, "hello");
}
#[tokio::test]
async fn result_participates_in_race_combinator() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#""callback-wins""#),
None,
None,
Some("cb-race"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("racer".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should succeed");
let slow = crate::future::DurableFuture::from_async(async {
tokio::time::sleep(Duration::from_secs(10)).await;
Ok("slow".to_owned())
});
let winner = ctx
.race([cb.result(), slow])
.await
.expect("race should resolve with the callback value");
assert_eq!(winner, "callback-wins");
}
#[tokio::test]
async fn result_participates_in_try_join_all_combinator() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#""from-callback""#),
None,
None,
Some("cb-join"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("joiner".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should succeed");
let other = crate::future::DurableFuture::from_async(async { Ok("plain".to_owned()) });
let values = ctx
.try_join_all([cb.result(), other])
.await
.expect("try_join_all should resolve");
assert_eq!(values, vec!["from-callback".to_owned(), "plain".to_owned()]);
}
#[tokio::test]
async fn replay_timed_out() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::TimedOut,
None,
None,
None,
Some("cb-456"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: Some(Duration::from_secs(5)),
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should return callback");
let err = cb.result().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::TimedOut)
));
}
#[tokio::test]
async fn replay_timed_out_on_heartbeat() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::TimedOut,
None,
Some("Callback.Heartbeat"),
Some("Callback timed out on heartbeat"),
Some("cb-790"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: Some(Duration::from_secs(10)),
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should return callback");
let err = cb.result().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::HeartbeatTimedOut)
));
}
#[tokio::test]
async fn replay_external_failure() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Failed,
None,
Some("ValidationError"),
Some("invalid input"),
Some("cb-fail"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should return callback");
let err = cb.result().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::ExternalFailure)
));
}
#[tokio::test]
async fn live_returns_assigned_id() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("my-cb".to_owned()),
timeout: Some(Duration::from_mins(1)),
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should succeed");
assert_eq!(cb.id(), "in-mem-cb-1");
}
fn ctx_with_hooks(
records: Vec<(String, CheckpointRecord)>,
hooks: &crate::plugin::test_support::OpHooks,
) -> DurableContext {
DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:123:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::new(InMemoryExecutionClient::new(vec![])),
"token-1".to_owned(),
Duration::ZERO,
0,
Some(hooks.plugins()),
)
}
async fn create_cb(ctx: &DurableContext) -> Result<Callback<String>, OperationError> {
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id: ctx.mint_id(),
name: Some("cb".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
exec.execute().await
}
fn hook_shapes(
hooks: &crate::plugin::test_support::OpHooks,
) -> Vec<(&'static str, bool, Option<crate::OperationStatus>)> {
hooks
.records()
.iter()
.map(|r| (r.hook, r.replay, r.status))
.collect()
}
#[tokio::test]
async fn callback_missing_id_fires_no_hook() {
let hooks = crate::plugin::test_support::OpHooks::default();
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_hooks(
vec![callback_record(
&wire,
CheckpointStatus::Started,
None,
None,
None,
None,
)],
&hooks,
);
let err = create_cb(&ctx).await.expect_err("the ID is missing");
assert_missing_callback_id_error(&err, &wire);
assert!(hooks.records().is_empty(), "{:?}", hooks.records());
}
#[tokio::test]
async fn live_callback_missing_id_fires_no_hook() {
let hooks = crate::plugin::test_support::OpHooks::default();
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
client.enqueue_checkpoint_response(crate::client::TestResponse::Success(vec![]));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token-1".to_owned(),
Duration::ZERO,
0,
Some(hooks.plugins()),
);
let wire = crate::engine::compute_wire_id_public("1");
let err = create_cb(&ctx).await.expect_err("the ID is missing");
assert_missing_callback_id_error(&err, &wire);
assert!(hooks.records().is_empty(), "{:?}", hooks.records());
}
#[tokio::test]
async fn running_callback_record_starts_replayed() {
for status in [CheckpointStatus::Started, CheckpointStatus::Pending] {
let hooks = crate::plugin::test_support::OpHooks::default();
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_hooks(
vec![callback_record(
&wire,
status.clone(),
None,
None,
None,
Some("cb-1"),
)],
&hooks,
);
create_cb(&ctx).await.expect("the callback is running");
let records = hooks.of("cb");
assert_eq!(records.len(), 1, "{status:?}: {records:?}");
let record = records.first().expect("one record");
assert_eq!((record.hook, record.replay), ("start", true), "{status:?}");
}
}
const FAILURE_STATUSES: [(CheckpointStatus, crate::OperationStatus); 4] = [
(CheckpointStatus::Failed, crate::OperationStatus::Failed),
(
CheckpointStatus::Cancelled,
crate::OperationStatus::Cancelled,
),
(CheckpointStatus::Stopped, crate::OperationStatus::Stopped),
(CheckpointStatus::TimedOut, crate::OperationStatus::TimedOut),
];
fn failure_field_cases(
wire: &str,
status: &CheckpointStatus,
) -> Vec<(String, CheckpointRecord)> {
let mut complete = callback_record(
wire,
status.clone(),
None,
Some("Declined"),
Some("no"),
Some("cb-1"),
);
complete.1.error_data = Some("d".to_owned());
complete.1.stack_trace = Some(vec!["frame-1".to_owned()]);
let mut absent = callback_record(
wire,
status.clone(),
None,
Some("Declined"),
None,
Some("cb-1"),
);
absent.1.error_data = Some("d".to_owned());
let empty = callback_record(
wire,
status.clone(),
None,
Some("Declined"),
Some(""),
Some("cb-1"),
);
let none = callback_record(wire, status.clone(), None, None, None, Some("cb-1"));
vec![complete, absent, empty, none]
}
fn carries_error(record: &CheckpointRecord) -> bool {
record.error_type.is_some()
|| record.error_message.is_some()
|| record.error_data.is_some()
|| record.stack_trace.is_some()
}
#[tokio::test]
async fn failed_callback_record_ends_replayed() {
let wire = crate::engine::compute_wire_id_public("1");
for (status, expected) in FAILURE_STATUSES {
for record in failure_field_cases(&wire, &status) {
let has_error = carries_error(&record.1);
let expected_message = match record.1.error_message.as_deref() {
Some(message) if !message.is_empty() => message.to_owned(),
_ if status == CheckpointStatus::TimedOut => "Callback timed out".to_owned(),
_ => "Callback failed".to_owned(),
};
let hooks = crate::plugin::test_support::OpHooks::default();
let ctx = ctx_with_hooks(vec![record], &hooks);
let cb = create_cb(&ctx).await.expect("the record replays");
assert_eq!(
hook_shapes(&hooks),
[("end", true, Some(expected))],
"{status:?}"
);
let err = cb.result().await.expect_err("the callback did not succeed");
let records = hooks.records();
assert_eq!(records.len(), 1, "result() fires no second hook");
let end = records.first().expect("the end hook");
if has_error {
assert_eq!(end.info_error.as_ref(), err.wire(), "{status:?}");
let message = end
.info_error
.as_ref()
.and_then(crate::WireError::error_message);
assert_eq!(message, Some(expected_message.as_str()), "{status:?}");
} else {
assert_eq!(end.info_error, None, "{status:?}");
}
}
}
}
#[tokio::test]
async fn updated_terminal_callback_record_ends_live_at_creation() {
let wire = crate::engine::compute_wire_id_public("1");
for (status, expected) in FAILURE_STATUSES {
let record = callback_record(
&wire,
status.clone(),
None,
Some("Declined"),
Some("no"),
Some("cb-1"),
);
let hooks = crate::plugin::test_support::OpHooks::default();
let operations = std::collections::BTreeMap::from([(
wire.clone(),
crate::plugin::OperationInfo::from_record(&wire, &record.1),
)]);
let factories: Vec<(&'static str, Arc<dyn crate::plugin::PluginFactory>)> =
vec![("test::OpHooks", Arc::new(hooks.clone()))];
let plugins = crate::plugin::create_plugins(&factories, None, || {
crate::plugin::InvocationInfo::new(
"r".to_owned(),
"arn".to_owned(),
"{}".to_owned(),
operations,
vec![wire.clone()],
None,
false,
None,
)
})
.expect("a factory is registered");
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:123:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(vec![record])),
Arc::new(InMemoryExecutionClient::new(vec![])),
"token-1".to_owned(),
Duration::ZERO,
0,
Some(plugins),
);
let cb = create_cb(&ctx).await.expect("the record replays");
assert_eq!(
hook_shapes(&hooks),
[("end", false, Some(expected))],
"{status:?}"
);
cb.result().await.expect_err("the callback did not succeed");
assert_eq!(hooks.records().len(), 1, "result() fires no second hook");
}
}
fn assert_missing_callback_id_error(err: &OperationError, wire: &str) {
assert!(
matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::Internal)
),
"expected Callback/Internal error, got {err:#}"
);
assert_eq!(err.operation_id(), Some(wire));
let chain = format!("{err:#}");
assert!(
chain.contains(wire),
"error should name the operation, got: {chain}"
);
}
#[tokio::test]
async fn replay_started_missing_callback_id_fails() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Started,
None,
None,
None,
None,
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("no-id".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("must fail on missing id");
assert_missing_callback_id_error(&err, &wire);
assert_eq!(err.status(), Some("STARTED"));
}
#[tokio::test]
async fn replay_started_empty_callback_id_fails() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Started,
None,
None,
None,
Some(""),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("must fail on empty id");
assert_missing_callback_id_error(&err, &wire);
}
#[tokio::test]
async fn replay_succeeded_missing_callback_id_fails() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#""hello""#),
None,
None,
None,
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("must fail on missing id");
assert_missing_callback_id_error(&err, &wire);
}
#[tokio::test]
async fn live_missing_callback_id_fails() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
client.enqueue_checkpoint_response(crate::client::TestResponse::Success(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let wire = op_id.wire().to_owned();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: Some("no-id-live".to_owned()),
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("must fail on missing id");
assert_missing_callback_id_error(&err, &wire);
assert_eq!(err.status(), Some("STARTED"));
}
#[tokio::test]
async fn id_stable_across_replay() {
let wire1 = crate::engine::compute_wire_id_public("1");
let wire2 = crate::engine::compute_wire_id_public("1");
assert_eq!(wire1, wire2);
}
#[tokio::test]
async fn wfcb_replay_succeeded() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#""done""#),
None,
None,
None,
)]);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("wfcb-test".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await.expect("should succeed");
assert_eq!(result, "done");
}
#[tokio::test]
async fn wfcb_replay_timed_out() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Failed,
None,
Some("Callback.Timeout"),
Some("Callback timed out"),
None,
)]);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::TimedOut)
));
}
#[tokio::test]
async fn wfcb_replay_heartbeat_timed_out() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Failed,
None,
Some("Callback.Heartbeat"),
Some("Callback timed out on heartbeat"),
None,
)]);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::HeartbeatTimedOut)
));
}
#[tokio::test]
async fn wfcb_submitter_failure_propagates() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("sub-fail-test".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async {
Err::<(), BoxError>("submitter exploded".into())
},
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
}
#[tokio::test]
async fn create_callback_spawn_executes() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token-1".to_owned(),
);
let future = ctx.create_callback::<String>().name("spawn-test").spawn();
let cb = future.await.expect("spawn should produce callback");
assert!(!cb.id().is_empty(), "expected an assigned callback id");
}
#[tokio::test]
async fn create_callback_ownership_rejects_foreign_task() {
let result = tokio::spawn(async {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
client,
"token-1".to_owned(),
);
let ctx_clone = ctx.clone();
let handle = tokio::spawn(async move {
let op_id = ctx_clone.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx_clone,
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
exec.execute().await
});
handle.await.unwrap()
})
.await
.unwrap();
assert!(result.is_err());
let err_msg = format!("{:#}", result.unwrap_err());
assert!(
err_msg.contains("task") || err_msg.contains("owner"),
"expected ownership error, got: {err_msg}"
);
}
fn assert_updates_include(
client: &InMemoryExecutionClient,
expected: &[(OperationType, OperationAction)],
) {
let updates = client.recorded_updates();
for (kind, action) in expected {
assert!(
updates
.iter()
.any(|u| u.r#type() == kind && u.action() == action),
"no {kind:?} {action:?} update in {updates:?}"
);
}
}
#[tokio::test]
async fn wfcb_submitter_executes_as_step_with_checkpoints() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let submitted = Arc::new(std::sync::atomic::AtomicBool::new(false));
let submitted_clone = Arc::clone(&submitted);
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("step-check".to_owned()),
timeout: None,
heartbeat: None,
submitter: move |_sc: StepContext, _id: String| {
submitted_clone.store(true, std::sync::atomic::Ordering::SeqCst);
async { Ok::<(), BoxError>(()) }
},
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
Arc::clone(ctx.coordinator()),
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert!(
submitted.load(std::sync::atomic::Ordering::SeqCst),
"submitter should have been invoked"
);
assert_updates_include(
&client,
&[
(OperationType::Context, OperationAction::Start),
(OperationType::Callback, OperationAction::Start),
(OperationType::Step, OperationAction::Start),
(OperationType::Step, OperationAction::Succeed),
],
);
}
#[tokio::test]
#[expect(clippy::too_many_lines)] async fn wfcb_submitter_replay_skips_re_execution() {
use std::sync::atomic::{AtomicBool, Ordering};
let wfcb_wire = crate::engine::compute_wire_id_public("1");
let cb_wire = crate::engine::compute_wire_id_public("1-1");
let step_wire = crate::engine::compute_wire_id_public("1-2");
let ctx = ctx_with_log(vec![
(
wfcb_wire.clone(),
CheckpointRecord {
id: wfcb_wire,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
(
cb_wire.clone(),
CheckpointRecord {
id: cb_wire,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: Some("cb-replay-test".to_owned()),
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
(
step_wire.clone(),
CheckpointRecord {
id: step_wire,
status: CheckpointStatus::Succeeded,
result: Some("null".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
]);
let submitter_called = Arc::new(AtomicBool::new(false));
let submitter_called_clone = Arc::clone(&submitter_called);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("replay-test".to_owned()),
timeout: None,
heartbeat: None,
submitter: move |_sc: StepContext, _id: String| {
submitter_called_clone.store(true, Ordering::SeqCst);
async { Ok::<(), BoxError>(()) }
},
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
Arc::clone(ctx.coordinator()),
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert!(
!submitter_called.load(Ordering::SeqCst),
"submitter should NOT be re-invoked during replay of a succeeded step"
);
}
#[tokio::test]
async fn wfcb_submitter_failure_checkpoints_step_and_propagates() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("fail-check".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async {
Err::<(), BoxError>("submission failed".into())
},
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_updates_include(
&client,
&[
(OperationType::Context, OperationAction::Start),
(OperationType::Callback, OperationAction::Start),
(OperationType::Step, OperationAction::Start),
(OperationType::Step, OperationAction::Retry),
],
);
}
#[tokio::test]
async fn wfcb_live_failure_carries_checkpointed_context() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let expected_wire_id = op_id.wire().to_owned();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("live-fail-context".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async {
Err::<(), BoxError>("submission failed".into())
},
submitter_retry: Some(Box::new(|_err: &crate::StepError, _attempt: u32| {
crate::RetryDecision::Stop
})),
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.unwrap_err();
assert!(
!ctx.admission()
.admits(std::slice::from_ref(&expected_wire_id)),
"a finished wait_for_callback child is abandoned"
);
assert_eq!(err.operation_id(), Some(expected_wire_id.as_str()));
assert_eq!(err.status(), Some("FAILED"));
let wire = err.wire().expect("live wfcb failure carries wire record");
assert_eq!(
wire.error_message(),
Some("submission failed"),
"the record carries the submitter step's recorded message: {wire:?}"
);
}
#[tokio::test]
async fn wfcb_submitter_receives_step_context_with_attempt() {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let received_attempt = Arc::new(std::sync::atomic::AtomicU32::new(0));
let received_attempt_clone = Arc::clone(&received_attempt);
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("attempt-check".to_owned()),
timeout: None,
heartbeat: None,
submitter: move |sc: StepContext, _id: String| {
received_attempt_clone.store(sc.attempt(), std::sync::atomic::Ordering::SeqCst);
async { Ok::<(), BoxError>(()) }
},
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let outcome = crate::driver::test_support::outcome_of(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
Arc::clone(ctx.coordinator()),
exec.execute(),
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
let attempt = received_attempt.load(std::sync::atomic::Ordering::SeqCst);
assert_eq!(
attempt, 1,
"submitter should receive attempt=1 on first invocation, got {attempt}"
);
}
#[derive(Debug)]
struct MarkerSerdes;
impl<T> Serdes<T> for MarkerSerdes
where
T: Serialize + DeserializeOwned + Send + 'static,
{
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(&self, value: T, _context: SerdesContext) -> Result<String, BoxError> {
Ok(format!("MARK:{}", serde_json::to_string(&value)?))
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(&self, wire: String, _context: SerdesContext) -> Result<T, BoxError> {
let body = wire
.strip_prefix("MARK:")
.ok_or_else(|| -> BoxError { "missing MARK: prefix".into() })?;
Ok(serde_json::from_str(body)?)
}
}
#[tokio::test]
async fn callback_per_op_serdes_decodes_non_json_payload() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#"MARK:"hello""#),
None,
None,
Some("cb-marker"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: MarkerSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should settle");
let value = cb.result().await.expect("per-op serdes should decode");
assert_eq!(value, "hello");
}
#[tokio::test]
async fn shared_probe_serdes_decodes_callback_payload() {
use crate::serdes::test_support::{HexEnvelopeSerdes, hex_envelope};
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Doc {
label: String,
nested: Vec<Vec<i64>>,
}
let doc = Doc {
label: "quote:\" backslash:\\ newline:\n tab:\t ünïcodé ☃".to_owned(),
nested: vec![vec![1, -2, i64::MIN], Vec::new()],
};
let wire = hex_envelope(&serde_json::to_string(&doc).expect("doc is JSON-able"));
assert!(serde_json::from_str::<Doc>(&wire).is_err());
let op_wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&op_wire,
CheckpointStatus::Succeeded,
Some(&wire),
None,
None,
Some("cb-probe"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<Doc, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: HexEnvelopeSerdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should settle");
let value = cb.result().await.expect("probe serdes should decode");
assert_eq!(value, doc);
}
#[tokio::test]
#[expect(clippy::indexing_slicing)] async fn callback_payload_never_resolves_file_references() {
use crate::serdes::FileSystemSerdes;
let tmp = std::env::temp_dir().join(format!(
"cb_no_file_refs_{}_{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock after epoch")
.as_nanos()
));
std::fs::create_dir_all(&tmp).expect("create temp base");
std::fs::write(tmp.join("secret.json"), r#""stolen contents""#).expect("plant file");
let filesystem_serdes = FileSystemSerdes::new(tmp.to_string_lossy().into_owned());
let payload = format!(
r#"{{"file":"{}","data":{{"status":"ready"}}}}"#,
tmp.join("secret.json").to_string_lossy()
);
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(&payload),
None,
None,
Some("cb-ext"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<serde_json::Value, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: filesystem_serdes,
_marker: std::marker::PhantomData,
};
let cb = exec.execute().await.expect("should settle");
let value = cb.result().await.expect("external payload decodes as data");
let _ = std::fs::remove_dir_all(&tmp);
assert_eq!(
value["data"],
serde_json::json!({"status": "ready"}),
"inline payload with a 'file' key must not be misparsed"
);
assert_eq!(
value["file"],
serde_json::json!(tmp.join("secret.json").to_string_lossy()),
"the 'file' key is plain data"
);
assert_ne!(
value,
serde_json::json!("stolen contents"),
"an external payload must never trigger a local file read"
);
}
#[tokio::test]
async fn callback_marker_payload_fails_without_serdes() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some(r#"MARK:"hello""#),
None,
None,
Some("cb-marker"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let callback = exec.execute().await.expect("creation succeeds");
let err = callback.result().await.unwrap_err();
assert!(matches!(
err.kind(),
OperationErrorKind::Callback(e)
if matches!(e.kind(), CallbackErrorKind::DeserializationFailed)
));
}
#[tokio::test]
async fn absent_callback_payload_decode_failure_comes_from_result() {
let wire = crate::engine::compute_wire_id_public("1");
let callback = create_from(callback_record(
&wire,
CheckpointStatus::Succeeded,
None,
None,
None,
Some("cb-1"),
))
.await
.expect("creation succeeds");
let err = callback.result().await.expect_err("null is not a String");
assert!(
matches!(
callback_kind(&err),
Some(CallbackErrorKind::DeserializationFailed)
),
"{err:?}"
);
}
#[derive(Clone, Default)]
pub(super) struct CountingFailingSerdes(pub(super) Arc<std::sync::atomic::AtomicUsize>);
impl CountingFailingSerdes {
fn decodes(&self) -> usize {
self.0.load(std::sync::atomic::Ordering::SeqCst)
}
}
impl Serdes<String> for CountingFailingSerdes {
fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Ok(value))
}
fn deserialize(
&self,
_wire: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
std::future::ready(Err("cannot decode".into()))
}
}
#[tokio::test]
async fn terminal_callback_decodes_once_at_creation() {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some("not json"),
None,
None,
Some("cb-1"),
)]);
let serdes = CountingFailingSerdes::default();
let callback = ctx
.create_callback::<String>()
.name("cb")
.serdes(serdes.clone())
.await
.expect("creation succeeds");
assert_eq!(serdes.decodes(), 1, "creation decodes the record once");
drop(callback);
assert_eq!(serdes.decodes(), 1, "dropping the callback decodes nothing");
assert!(
ctx.coordinator().fatal_error().is_none(),
"an unread decode failure fails nothing"
);
}
#[tokio::test]
async fn wfcb_per_op_serdes_decodes_raw_payload() {
let ctx = ctx_with_log(wfcb_settled_inner(r#"MARK:"wfcb""#));
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("wfcb-serdes".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: MarkerSerdes,
_marker: std::marker::PhantomData,
};
let result = exec
.execute()
.await
.expect("wfcb should decode via its serdes");
assert_eq!(result, "wfcb");
}
fn wfcb_settled_inner(payload: &str) -> Vec<(String, CheckpointRecord)> {
let wfcb_wire = crate::engine::compute_wire_id_public("1");
let cb_wire = crate::engine::compute_wire_id_public("1-1");
let step_wire = crate::engine::compute_wire_id_public("1-2");
vec![
callback_record(
&wfcb_wire,
CheckpointStatus::Started,
None,
None,
None,
None,
),
callback_record(
&cb_wire,
CheckpointStatus::Succeeded,
Some(payload),
None,
None,
Some("cb-wfcb"),
),
callback_record(
&step_wire,
CheckpointStatus::Succeeded,
Some("null"),
None,
None,
None,
),
]
}
fn ctx_and_client_with_log(
records: Vec<(String, CheckpointRecord)>,
) -> (DurableContext, Arc<InMemoryExecutionClient>) {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:123:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
(ctx, client)
}
fn outer_succeed_payloads(client: &InMemoryExecutionClient) -> Vec<Option<String>> {
let wfcb_wire = crate::engine::compute_wire_id_public("1");
client
.recorded_updates()
.into_iter()
.filter(|u| u.id() == wfcb_wire && *u.action() == OperationAction::Succeed)
.map(|u| u.payload().map(ToOwned::to_owned))
.collect()
}
#[derive(Debug, Default)]
struct Calls {
deserialize_inputs: std::sync::Mutex<Vec<String>>,
serializes: std::sync::atomic::AtomicUsize,
}
impl Calls {
fn deserialize_inputs(&self) -> Vec<String> {
self.deserialize_inputs.lock().unwrap().clone()
}
fn serializes(&self) -> usize {
self.serializes.load(std::sync::atomic::Ordering::SeqCst)
}
}
struct Counting<S> {
inner: S,
calls: Arc<Calls>,
}
impl<S> Counting<S> {
fn new(inner: S) -> (Self, Arc<Calls>) {
let calls = Arc::new(Calls::default());
let serdes = Self {
inner,
calls: Arc::clone(&calls),
};
(serdes, calls)
}
}
impl<T: Send + 'static, S: Serdes<T>> Serdes<T> for Counting<S> {
async fn serialize(&self, value: T, context: SerdesContext) -> Result<String, BoxError> {
self.calls
.serializes
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
self.inner.serialize(value, context).await
}
async fn deserialize(&self, wire: String, context: SerdesContext) -> Result<T, BoxError> {
self.calls
.deserialize_inputs
.lock()
.unwrap()
.push(wire.clone());
self.inner.deserialize(wire, context).await
}
}
async fn run_live_wfcb<O, S>(
payload: &str,
serdes: S,
) -> (Result<O, OperationError>, Vec<Option<String>>)
where
O: Send + 'static,
S: Serdes<O>,
{
let (ctx, client) = ctx_and_client_with_log(wfcb_settled_inner(payload));
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<O, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("wfcb-raw".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes,
_marker: std::marker::PhantomData,
};
let result = exec.execute().await;
(result, outer_succeed_payloads(&client))
}
#[tokio::test]
async fn wfcb_records_raw_payload_and_decodes_once() {
let (serdes, calls) = Counting::new(crate::serdes::JsonSerdes);
let (result, recorded) = run_live_wfcb::<String, _>(r#""hi""#, serdes).await;
assert_eq!(result.expect("decodes"), "hi");
assert_eq!(recorded, [Some(r#""hi""#.to_owned())]);
assert_eq!(calls.deserialize_inputs(), [r#""hi""#]);
assert_eq!(calls.serializes(), 0);
let (serdes, calls) = Counting::new(MarkerSerdes);
let (result, recorded) = run_live_wfcb::<String, _>(r#"MARK:"wfcb""#, serdes).await;
assert_eq!(result.expect("decodes"), "wfcb");
assert_eq!(recorded, [Some(r#"MARK:"wfcb""#.to_owned())]);
assert_eq!(calls.deserialize_inputs(), [r#"MARK:"wfcb""#]);
assert_eq!(calls.serializes(), 0);
}
async fn replay_wfcb_succeeded<O, S>(
payload: Option<&str>,
serdes: S,
) -> Result<O, OperationError>
where
O: Send + 'static,
S: Serdes<O>,
{
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
payload,
None,
None,
None,
)]);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<O, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("wfcb-replay".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async {
Err::<(), BoxError>("a replay must not run the submitter".into())
},
submitter_retry: None,
serdes,
_marker: std::marker::PhantomData,
};
exec.execute().await
}
#[tokio::test]
async fn wfcb_replay_applies_user_serdes() {
let (serdes, calls) = Counting::new(MarkerSerdes);
let value: String = replay_wfcb_succeeded(Some(r#"MARK:"wfcb""#), serdes)
.await
.expect("decodes");
assert_eq!(value, "wfcb");
assert_eq!(calls.deserialize_inputs(), [r#"MARK:"wfcb""#]);
assert_eq!(calls.serializes(), 0);
let (serdes, calls) = Counting::new(crate::serdes::JsonSerdes);
let value: String = replay_wfcb_succeeded(Some(r#""hi""#), serdes)
.await
.expect("decodes");
assert_eq!(value, "hi");
assert_eq!(calls.deserialize_inputs(), [r#""hi""#]);
assert_eq!(calls.serializes(), 0);
let (serdes, calls) = Counting::new(crate::serdes::JsonSerdes);
let value: Option<String> = replay_wfcb_succeeded(None, serdes).await.expect("decodes");
assert_eq!(value, None);
assert_eq!(calls.deserialize_inputs(), ["null"]);
assert_eq!(calls.serializes(), 0);
let value: String = replay_wfcb_succeeded(None, PassthroughSerdes)
.await
.expect("decodes");
assert_eq!(value, "null");
}
#[tokio::test]
async fn wfcb_marked_decode_failure_ends_invocation() {
struct MarkedDecodeFailure;
impl Serdes<String> for MarkedDecodeFailure {
fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Ok(value))
}
fn deserialize(
&self,
_wire: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Err(
crate::serdes::RetryableSerdesError::new("transient").into()
))
}
}
let (ctx, client) = ctx_and_client_with_log(wfcb_settled_inner("approved"));
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: Some("wfcb-marked".to_owned()),
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: MarkedDecodeFailure,
_marker: std::marker::PhantomData,
};
let signal = Arc::clone(ctx.coordinator());
crate::driver::test_support::expect_serdes_fault(
ctx.clone(),
Arc::clone(ctx.task_ownership()),
signal,
exec.execute(),
"serdes deserialize failed",
)
.await;
assert_eq!(
outer_succeed_payloads(&client),
[Some("approved".to_owned())]
);
}
#[tokio::test]
async fn create_callback_passthrough_returns_raw_payload() {
async fn read<S: Serdes<String>>(serdes: S) -> Result<String, OperationError> {
let wire = crate::engine::compute_wire_id_public("1");
let ctx = ctx_with_log(vec![callback_record(
&wire,
CheckpointStatus::Succeeded,
Some("approved"),
None,
None,
Some("cb-raw"),
)]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes,
_marker: std::marker::PhantomData,
};
exec.execute().await?.result().await
}
assert_eq!(read(PassthroughSerdes).await.expect("raw"), "approved");
let err = read(crate::serdes::JsonSerdes).await.expect_err("not JSON");
assert!(matches!(
callback_kind(&err),
Some(CallbackErrorKind::DeserializationFailed)
));
}
fn callback_kind(err: &OperationError) -> Option<&CallbackErrorKind> {
match err.kind() {
OperationErrorKind::Callback(e) => Some(e.kind()),
_ => None,
}
}
async fn replay_wfcb_outer(
error_type: Option<&str>,
error_message: Option<&str>,
error_data: Option<&str>,
) -> OperationError {
let wire = crate::engine::compute_wire_id_public("1");
let mut record = callback_record(
&wire,
CheckpointStatus::Failed,
None,
error_type,
error_message,
None,
);
record.1.error_data = error_data.map(String::from);
let ctx = ctx_with_log(vec![record]);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
exec.execute()
.await
.expect_err("a FAILED record replays its error")
}
#[tokio::test]
async fn wfcb_replay_maps_fixed_types() {
let cases: [(Option<&str>, Option<&str>, &str); 10] = [
(Some("Callback.ExternalFailure"), None, "ExternalFailure"),
(Some("Callback.Heartbeat"), None, "HeartbeatTimedOut"),
(Some("Callback.Internal"), None, "Internal"),
(
Some("Callback.ExternalFailure"),
Some(r#"{"code":7}"#),
"ExternalFailure",
),
(Some("Callback.SubmitterFailed"), None, "SubmitterFailed"),
(
Some("Callback.DeserializationFailed"),
None,
"DeserializationFailed",
),
(Some("Callback.Timeout"), None, "TimedOut"),
(Some("StepError"), None, "ExternalFailure"),
(Some("PaymentDeclined"), None, "ExternalFailure"),
(None, None, "ExternalFailure"),
];
for (error_type, data, expected) in cases {
let err = replay_wfcb_outer(error_type, Some("recorded"), data).await;
let kind = callback_kind(&err).map(|k| format!("{k:?}"));
assert_eq!(kind.as_deref(), Some(expected), "{error_type:?}: {err:?}");
let wire = err.wire().expect("a replayed failure carries its record");
assert_eq!(wire.error_type(), error_type);
assert_eq!(wire.error_data(), data);
assert_eq!(err.status(), Some("FAILED"));
let timeout = matches!(expected, "TimedOut" | "HeartbeatTimedOut");
let source = std::error::Error::source(&err).and_then(std::error::Error::source);
assert_eq!(source.is_none(), timeout, "{expected}: {err:?}");
}
}
fn external_failure(wire: Option<crate::error::WireError>) -> OperationError {
let source = wire.clone().map(crate::error::ReplayedFailure::source_from);
let err = OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::ExternalFailure,
source,
)));
match wire {
Some(wire) => err.with_wire(wire),
None => err,
}
}
fn submitter_failure(source: BoxError) -> OperationError {
let step =
OperationError::from_kind(OperationErrorKind::Step(crate::error::StepError::new(
crate::error::StepErrorKind::ExecutionFailed,
Some(source),
)));
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
CallbackErrorKind::SubmitterFailed,
Some(Box::new(step)),
)))
}
fn callback_failure(kind: CallbackErrorKind, message: &str) -> OperationError {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind,
Some(message.to_owned().into()),
)))
}
#[test]
fn extract_wire_error_records_fixed_sdk_types() {
let external = crate::error::WireError::new(Some("PaymentDeclined"), Some("card declined"))
.with_error_data(Some(r#"{"code":7}"#));
let wire = extract_wire_error(&external_failure(Some(external)));
assert_eq!(wire.error_type(), Some("Callback.ExternalFailure"));
assert_eq!(wire.error_message(), Some("card declined"));
assert_eq!(wire.error_data(), Some(r#"{"code":7}"#));
let fallback = |data: Option<&str>| {
crate::error::WireError::new(Some("Callback.ExternalFailure"), Some("Callback failed"))
.with_error_data(data)
};
let absent = crate::error::WireError::new(Some("PaymentDeclined"), None::<&str>)
.with_error_data(Some("d"));
assert_eq!(
extract_wire_error(&external_failure(Some(absent))),
fallback(Some("d"))
);
let empty = crate::error::WireError::new(Some("PaymentDeclined"), Some(""));
assert_eq!(
extract_wire_error(&external_failure(Some(empty))),
fallback(None)
);
assert_eq!(extract_wire_error(&external_failure(None)), fallback(None));
let wire = extract_wire_error(&submitter_failure("submission failed".into()));
assert_eq!(wire.error_type(), Some("Callback.SubmitterFailed"));
assert_eq!(
wire.error_message(),
Some("submitter failed: operation error: step: execution failed: submission failed")
);
let typed = crate::TypedError::with_type("PaymentError", "declined");
let wire = extract_wire_error(&submitter_failure(Box::new(typed)));
assert_eq!(wire.error_type(), Some("Callback.SubmitterFailed"));
assert_eq!(
wire.error_message(),
Some("submitter failed: operation error: step: execution failed: declined")
);
let wire = extract_wire_error(&callback_failure(CallbackErrorKind::Internal, "oops"));
assert_eq!(wire.error_type(), Some("Callback.Internal"));
assert_eq!(wire.error_message(), Some("internal error: oops"));
let typed = crate::TypedError::with_type("PaymentError", "declined");
let other = OperationError::from_kind(OperationErrorKind::ChildContext(
ChildContextError::new(ChildContextErrorKind::ChildFailed, Some(Box::new(typed))),
));
let wire = extract_wire_error(&other);
assert_eq!(wire.error_type(), Some("Callback.Internal"));
assert_eq!(
wire.error_message(),
Some("operation error: child_context: child failed: declined")
);
let recorded = || {
crate::error::recorded_step_error(crate::error::WireError::new(
Some("InnerType"),
Some("inner"),
))
};
let callback_over = |kind: CallbackErrorKind, source: crate::error::Source| {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind,
Some(source),
)))
};
let rows = [
(
callback_over(CallbackErrorKind::SubmitterFailed, Box::new(recorded())),
"Callback.SubmitterFailed",
),
(
callback_over(CallbackErrorKind::Internal, Box::new(recorded())),
"Callback.Internal",
),
(
callback_over(
CallbackErrorKind::DeserializationFailed,
crate::error::ContextualError::source_from(
"deserialize callback result",
recorded(),
),
),
"Callback.DeserializationFailed",
),
(recorded(), "Callback.Internal"),
];
for (err, expected_type) in &rows {
let wire = extract_wire_error(err);
assert_eq!(wire.error_type(), Some(*expected_type), "{err:?}");
assert_eq!(wire.error_message(), Some("inner"), "{err:?}");
}
let wire = extract_wire_error(&callback_failure(
CallbackErrorKind::DeserializationFailed,
"bad payload",
));
assert_eq!(wire.error_type(), Some("Callback.DeserializationFailed"));
assert_eq!(
wire.error_message(),
Some("deserialization failed: bad payload")
);
assert_timeout_wire(|| CallbackErrorKind::TimedOut, "Callback.Timeout");
assert_timeout_wire(
|| CallbackErrorKind::HeartbeatTimedOut,
"Callback.Heartbeat",
);
}
fn assert_timeout_wire(kind_of: impl Fn() -> CallbackErrorKind, expected_type: &str) {
let timeout = |wire: crate::error::WireError| {
OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind_of(),
None,
)))
.with_wire(wire)
};
let recorded = crate::error::WireError::new(Some("Recorded.Type"), Some("m"))
.with_error_data(Some("d"));
assert_eq!(
extract_wire_error(&timeout(recorded)),
crate::error::WireError::new(Some(expected_type), Some("m")).with_error_data(Some("d")),
"{expected_type}"
);
for message in [None, Some("")] {
let recorded = crate::error::WireError::new(Some("Recorded.Type"), message)
.with_error_data(Some("d"));
assert_eq!(
extract_wire_error(&timeout(recorded)),
crate::error::WireError::new(Some(expected_type), Some("Callback timed out"))
.with_error_data(Some("d")),
"{expected_type} {message:?}"
);
}
let bare = OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind_of(),
None,
)));
assert_eq!(
extract_wire_error(&bare),
crate::error::WireError::new(Some(expected_type), Some("Callback timed out")),
"{expected_type} bare"
);
}
async fn settle_live(err: OperationError) -> (OperationError, crate::error::WireError) {
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let wire_id = crate::engine::compute_wire_id_public("1");
let after = WfcbAfter {
ctx,
wire_id: wire_id.clone(),
name: None,
};
let live = after
.settle(
None,
crate::plugin::Ran::from_result(Err(err), extract_wire_error),
)
.await
.expect_err("a failed body fails the operation");
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.id() == wire_id && *u.action() == OperationAction::Fail)
.expect("the outer FAIL was written");
let object = fail.error().expect("the FAIL carries an error");
let recorded = crate::error::WireError::new(object.error_type(), object.error_message())
.with_error_data(object.error_data());
(live, recorded)
}
fn assert_live_equals_replay(live: &OperationError) {
let wire = live
.wire()
.expect("the live error carries the outer record");
let replayed = wfcb_failed_error(
wire.clone(),
live.operation_id()
.expect("the live error names the operation"),
live.status().expect("the live error has a status"),
);
assert_eq!(
callback_kind(live).map(|k| format!("{k:?}")),
callback_kind(&replayed).map(|k| format!("{k:?}")),
);
assert_eq!(live.operation_id(), replayed.operation_id());
assert_eq!(live.status(), replayed.status());
assert_eq!(live.wire(), replayed.wire());
}
#[tokio::test]
async fn wfcb_live_external_failure_hides_external_type() {
let external = crate::error::WireError::new(Some("PaymentDeclined"), Some("card declined"))
.with_error_data(Some(r#"{"code":7}"#));
let (live, recorded) = settle_live(external_failure(Some(external))).await;
assert_eq!(live.wire(), Some(&recorded));
assert_eq!(recorded.error_type(), Some("Callback.ExternalFailure"));
let mut link: Option<&(dyn std::error::Error + 'static)> = Some(&live);
while let Some(e) = link {
if let Some(replayed) = e.downcast_ref::<crate::error::ReplayedFailure>() {
assert_ne!(replayed.error_type(), Some("PaymentDeclined"), "{live:?}");
}
link = e.source();
}
assert!(
matches!(
callback_kind(&live),
Some(CallbackErrorKind::ExternalFailure)
),
"{live:?}"
);
assert_live_equals_replay(&live);
}
#[tokio::test]
async fn wfcb_live_errors_equal_their_replay() {
let not_callback = || {
OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
ChildContextErrorKind::ChildFailed,
Some("inner".into()),
)))
};
let (live, _) = settle_live(not_callback()).await;
assert!(
matches!(callback_kind(&live), Some(CallbackErrorKind::Internal)),
"{live:?}"
);
let source = std::error::Error::source(&live)
.and_then(std::error::Error::source)
.and_then(|s| s.downcast_ref::<OperationError>())
.expect("the source is the original body error");
assert!(matches!(source.kind(), OperationErrorKind::ChildContext(_)));
assert_live_equals_replay(&live);
let (live, _) = settle_live(submitter_failure("submission failed".into())).await;
assert!(
matches!(
callback_kind(&live),
Some(CallbackErrorKind::SubmitterFailed)
),
"{live:?}"
);
let step = std::error::Error::source(&live)
.and_then(std::error::Error::source)
.and_then(|s| s.downcast_ref::<OperationError>())
.expect("the live source is the submitter step error");
assert!(matches!(step.kind(), OperationErrorKind::Step(_)));
assert_live_equals_replay(&live);
for kind in [
CallbackErrorKind::Internal,
CallbackErrorKind::DeserializationFailed,
] {
let (live, _) = settle_live(callback_failure(kind, "cause")).await;
assert!(format!("{live:#}").contains("cause"), "{live:#}");
assert_live_equals_replay(&live);
}
for kind in [
CallbackErrorKind::TimedOut,
CallbackErrorKind::HeartbeatTimedOut,
] {
let err = OperationError::from_kind(OperationErrorKind::Callback(CallbackError::new(
kind, None,
)));
let (live, _) = settle_live(err).await;
assert_live_equals_replay(&live);
}
}
async fn create_from(
record: (String, CheckpointRecord),
) -> Result<Callback<String>, OperationError> {
let ctx = ctx_with_log(vec![record]);
let op_id = ctx.mint_id();
let exec: CreateCallbackExecution<String, _> = CreateCallbackExecution {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
sub_type: crate::OperationSubType::CALLBACK,
plugin_name: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
exec.execute().await
}
fn callback_with(
status: CheckpointStatus,
error_type: Option<&str>,
) -> (String, CheckpointRecord) {
let wire = crate::engine::compute_wire_id_public("1");
callback_record(
&wire,
status,
None,
error_type,
Some("recorded"),
Some("cb-1"),
)
}
fn replayed_source_wire(err: &OperationError) -> Option<crate::error::WireError> {
let mut link: Option<&(dyn std::error::Error + 'static)> = Some(err);
while let Some(e) = link {
if let Some(replayed) = e.downcast_ref::<crate::error::ReplayedFailure>() {
return Some(replayed.wire().clone());
}
link = e.source();
}
None
}
async fn both_failure_errors(record: (String, CheckpointRecord)) -> [OperationError; 2] {
let cb = create_from(record.clone())
.await
.expect("a failed callback settles");
let from_result = cb.result().await.expect_err("the callback failed");
let cb = create_from(record)
.await
.expect("a failed callback settles");
let from_spawn = cb.spawn_result().await.expect_err("the callback failed");
[from_result, from_spawn]
}
async fn wfcb_over_callback(
callback: (String, CheckpointRecord),
) -> (OperationError, crate::error::WireError) {
let wfcb_wire = crate::engine::compute_wire_id_public("1");
let step_wire = crate::engine::compute_wire_id_public("1-2");
let log = CheckpointLog::from_records(vec![
callback_record(
&wfcb_wire,
CheckpointStatus::Started,
None,
None,
None,
None,
),
callback,
callback_record(
&step_wire,
CheckpointStatus::Succeeded,
Some("null"),
None,
None,
None,
),
]);
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("the callback failed");
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.id() == wfcb_wire && *u.action() == OperationAction::Fail)
.expect("the outer FAIL was written");
let error = fail.error().expect("the FAIL carries an error");
let wire = crate::error::WireError::new(error.error_type(), error.error_message())
.with_error_data(error.error_data());
(err, wire)
}
async fn assert_wfcb_external_failure(status: &CheckpointStatus) {
let cb_wire = crate::engine::compute_wire_id_public("1-1");
let mut callback = callback_record(
&cb_wire,
status.clone(),
None,
Some("Declined"),
None,
Some("cb-1"),
);
callback.1.error_data = Some("d".to_owned());
let (live, recorded) = wfcb_over_callback(callback).await;
let expected =
crate::error::WireError::new(Some(WFCB_EXTERNAL_FAILURE), Some("Callback failed"))
.with_error_data(Some("d"));
assert_eq!(recorded, expected, "{status:?}");
assert_eq!(live.wire(), Some(&expected), "{status:?}");
assert!(matches!(
callback_kind(&live),
Some(CallbackErrorKind::ExternalFailure)
));
let replayed = replay_wfcb_outer(
recorded.error_type(),
recorded.error_message(),
recorded.error_data(),
)
.await;
assert_eq!(replayed.wire(), live.wire(), "{status:?}");
assert!(matches!(
callback_kind(&replayed),
Some(CallbackErrorKind::ExternalFailure)
));
}
#[tokio::test]
async fn replay_cancelled_and_stopped_settle_as_external_failure() {
let wire = crate::engine::compute_wire_id_public("1");
let statuses = [
CheckpointStatus::Failed,
CheckpointStatus::Cancelled,
CheckpointStatus::Stopped,
];
for status in &statuses {
let mut record = callback_record(
&wire,
status.clone(),
None,
Some("PaymentDeclined"),
Some("card declined"),
Some("cb-1"),
);
record.1.error_data = Some("d".to_owned());
record.1.stack_trace = Some(vec!["frame-1".to_owned(), "frame-2".to_owned()]);
let expected =
crate::error::WireError::new(Some("PaymentDeclined"), Some("card declined"))
.with_error_data(Some("d"))
.with_stack_trace(vec!["frame-1".to_owned(), "frame-2".to_owned()]);
for err in both_failure_errors(record).await {
assert!(
matches!(
callback_kind(&err),
Some(CallbackErrorKind::ExternalFailure)
),
"{status:?}: {err:?}"
);
assert_eq!(err.status(), Some(status.wire_str()));
assert_eq!(err.wire(), Some(&expected), "{status:?}");
assert_eq!(replayed_source_wire(&err).as_ref(), Some(&expected));
}
let cases: [(Option<&str>, Option<&str>); 5] = [
(Some("Declined"), None),
(Some("Declined"), Some("")),
(None, None),
(Some("Callback.Timeout"), Some("")),
(Some("Callback.Heartbeat"), None),
];
for (error_type, message) in cases {
let record = callback_record(
&wire,
status.clone(),
None,
error_type,
message,
Some("cb-1"),
);
let expected = crate::error::WireError::new(error_type, Some("Callback failed"));
for err in both_failure_errors(record).await {
assert!(
matches!(
callback_kind(&err),
Some(CallbackErrorKind::ExternalFailure)
),
"{status:?} {error_type:?}: {err:?}"
);
assert_eq!(err.wire(), Some(&expected), "{status:?} {error_type:?}");
assert_eq!(replayed_source_wire(&err).as_ref(), Some(&expected));
}
}
assert_wfcb_external_failure(status).await;
}
let err = create_from(callback_with(CheckpointStatus::Ready, None))
.await
.expect_err("a READY callback is unexpected");
assert!(
matches!(callback_kind(&err), Some(CallbackErrorKind::Internal)),
"{err:?}"
);
let err = create_from(callback_with(
CheckpointStatus::Unknown("PAUSED".to_owned()),
None,
))
.await
.expect_err("a PAUSED callback is unrecognized");
let OperationErrorKind::NonDeterministicExecution(nde) = err.kind() else {
unreachable!("expected NonDeterministicExecution, got {:?}", err.kind());
};
let crate::error::NonDeterministicExecutionErrorKind::UnrecognizedStatus(details) =
nde.kind()
else {
unreachable!("expected UnrecognizedStatus, got {:?}", nde.kind());
};
assert_eq!(details.status(), "PAUSED");
}
#[tokio::test]
async fn replay_timed_out_reads_heartbeat_type() {
fresh_timed_out_callbacks_keep_their_record().await;
heartbeat_timeout_in_wfcb_keeps_the_callback_message().await;
}
async fn both_timeout_errors(record: (String, CheckpointRecord)) -> [OperationError; 2] {
let cb = create_from(record.clone())
.await
.expect("a TIMED_OUT callback settles");
let from_result = cb.result().await.expect_err("the callback timed out");
let cb = create_from(record)
.await
.expect("a TIMED_OUT callback settles");
let from_spawn = cb.spawn_result().await.expect_err("the callback timed out");
[from_result, from_spawn]
}
async fn fresh_timed_out_callbacks_keep_their_record() {
let wire = crate::engine::compute_wire_id_public("1");
let mut record = callback_record(
&wire,
CheckpointStatus::TimedOut,
None,
Some("Callback.Timeout"),
Some("the service's message"),
Some("cb-1"),
);
record.1.error_data = Some("d".to_owned());
record.1.stack_trace = Some(vec!["frame-1".to_owned(), "frame-2".to_owned()]);
for err in both_timeout_errors(record).await {
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Callback.Timeout"));
assert_eq!(wire.error_message(), Some("the service's message"));
assert_eq!(wire.error_data(), Some("d"));
assert_eq!(wire.stack_trace(), ["frame-1", "frame-2"]);
assert!(matches!(
callback_kind(&err),
Some(CallbackErrorKind::TimedOut)
));
}
let cases: [(Option<&str>, Option<&str>, &str); 3] = [
(Some("Callback.Heartbeat"), None, "HeartbeatTimedOut"),
(Some("Callback.Timeout"), Some(""), "TimedOut"),
(None, None, "TimedOut"),
];
for (error_type, message, expected) in cases {
let wire = crate::engine::compute_wire_id_public("1");
let record = callback_record(
&wire,
CheckpointStatus::TimedOut,
None,
error_type,
message,
Some("cb-1"),
);
for err in both_timeout_errors(record).await {
let kind = callback_kind(&err).map(|k| format!("{k:?}"));
assert_eq!(kind.as_deref(), Some(expected), "{error_type:?}");
assert!(
std::error::Error::source(&err)
.and_then(std::error::Error::source)
.is_none()
);
let wire = err.wire().expect("the error carries a wire record");
assert_eq!(wire.error_type(), error_type, "{error_type:?}");
assert_eq!(
wire.error_message(),
Some("Callback timed out"),
"{error_type:?}"
);
}
}
}
async fn heartbeat_timeout_in_wfcb_keeps_the_callback_message() {
let wfcb_wire = crate::engine::compute_wire_id_public("1");
let cb_wire = crate::engine::compute_wire_id_public("1-1");
let step_wire = crate::engine::compute_wire_id_public("1-2");
let mut timed_out = callback_record(
&cb_wire,
CheckpointStatus::TimedOut,
None,
Some("Callback.Heartbeat"),
Some("Callback timed out on heartbeat"),
Some("cb-1"),
);
timed_out.1.error_data = Some("d".to_owned());
let log = CheckpointLog::from_records(vec![
callback_record(
&wfcb_wire,
CheckpointStatus::Started,
None,
None,
None,
None,
),
timed_out,
callback_record(
&step_wire,
CheckpointStatus::Succeeded,
Some("null"),
None,
None,
None,
),
]);
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
let err = exec.execute().await.expect_err("the heartbeat timed out");
assert!(
matches!(
callback_kind(&err),
Some(CallbackErrorKind::HeartbeatTimedOut)
),
"{err:?}"
);
let expected = crate::error::WireError::new(
Some("Callback.Heartbeat"),
Some("Callback timed out on heartbeat"),
)
.with_error_data(Some("d"));
assert_eq!(err.wire(), Some(&expected));
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.id() == wfcb_wire && *u.action() == OperationAction::Fail)
.expect("the outer FAIL was written");
let error = fail.error().expect("the FAIL carries an error");
assert_eq!(error.error_type(), Some("Callback.Heartbeat"));
assert_eq!(
error.error_message(),
Some("Callback timed out on heartbeat")
);
assert_eq!(error.error_data(), Some("d"));
let replayed = replay_wfcb_outer(
error.error_type(),
error.error_message(),
error.error_data(),
)
.await;
assert_eq!(replayed.wire(), err.wire());
assert!(matches!(
callback_kind(&replayed),
Some(CallbackErrorKind::HeartbeatTimedOut)
));
}
#[tokio::test]
async fn wfcb_non_step_submitter_fatal_refuses_terminal_write() {
let wfcb_wire = crate::engine::compute_wire_id_public("1");
let cb_wire = crate::engine::compute_wire_id_public("1-1");
let step_wire = crate::engine::compute_wire_id_public("1-2");
let mut step = callback_record(
&step_wire,
CheckpointStatus::Succeeded,
None,
None,
None,
None,
);
step.1.op_type = Some("Wait".to_owned());
let log = CheckpointLog::from_records(vec![
callback_record(
&wfcb_wire,
CheckpointStatus::Started,
None,
None,
None,
None,
),
callback_record(
&cb_wire,
CheckpointStatus::Started,
None,
None,
None,
Some("cb-1"),
),
step,
]);
let client = Arc::new(InMemoryExecutionClient::new(vec![]));
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
let op_id = ctx.mint_id();
let exec = WaitForCallbackExecution::<String, _, _> {
ctx: ctx.clone(),
op_id,
name: None,
timeout: None,
heartbeat: None,
submitter: |_sc: StepContext, _id: String| async { Ok::<(), BoxError>(()) },
submitter_retry: None,
serdes: crate::serdes::JsonSerdes,
_marker: std::marker::PhantomData,
};
assert!(
tokio::time::timeout(Duration::from_millis(50), exec.execute())
.await
.is_err(),
"the outer FAIL is refused, so the operation never resolves"
);
let terminal: Vec<_> = client
.recorded_updates()
.into_iter()
.filter(|u| {
u.id() == wfcb_wire
&& matches!(u.action(), OperationAction::Fail | OperationAction::Succeed)
})
.collect();
assert!(terminal.is_empty(), "{terminal:?}");
let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
assert_eq!(fatal.error_type, "NonDeterministicExecutionError");
}
#[tokio::test]
async fn preserved_wait_for_callback_failure_has_flag() {
for (depth, flag) in [(1, Some(true)), (0, None)] {
let (ctx, client) = DurableContext::test_context_with_depth(depth);
let result = ctx
.wait_for_callback::<String>(|_step, _callback_id| async {
Err("cannot submit".into())
})
.name("approve")
.submitter_retry(|_err, _attempt| crate::RetryDecision::Stop)
.await;
assert!(result.is_err());
let fail = client
.recorded_updates()
.into_iter()
.find(|u| u.name() == Some("approve") && u.action() == &OperationAction::Fail)
.expect("the FAIL of the wait_for_callback");
assert_eq!(
fail.context_options()
.and_then(aws_sdk_lambda::types::ContextOptions::replay_children),
flag,
"depth {depth}"
);
}
}
#[tokio::test]
async fn callback_own_record_keeps_external_classification() {
let cb = create_from(callback_with(
CheckpointStatus::Failed,
Some("Callback.SubmitterFailed"),
))
.await
.expect("a FAILED callback settles");
let err = cb.result().await.expect_err("the callback failed");
assert!(
matches!(
callback_kind(&err),
Some(CallbackErrorKind::ExternalFailure)
),
"{err:?}"
);
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some("Callback.SubmitterFailed")
);
}
}
#[cfg(test)]
mod coordinator_tests {
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{
ScriptedClient, drive_handler, named, operation_like, scripted_ctx, wire_err,
};
use aws_sdk_lambda::types::{
CallbackDetails, ErrorObject, Operation, OperationAction, OperationStatus, OperationType,
OperationUpdate,
};
use std::sync::Arc;
fn started(start: &OperationUpdate) -> Operation {
operation_like(start, OperationStatus::Started)
.callback_details(CallbackDetails::builder().callback_id("cb-1").build())
.build()
.expect("all required fields set")
}
fn client_completing(
completed: impl Fn(&OperationUpdate) -> Operation + Send + Sync + 'static,
) -> ScriptedClient {
ScriptedClient::new(move |updates, history| {
let mut ops = Vec::new();
for update in updates {
if update.r#type() == &OperationType::Callback
&& update.action() == &OperationAction::Start
{
ops.push(started(update));
}
}
if named(updates, "s", &OperationAction::Succeed).is_some()
&& let Some(start) = named(history, "cb", &OperationAction::Start)
{
ops.push(completed(start));
}
ops
})
}
fn succeeded(start: &OperationUpdate) -> Operation {
operation_like(start, OperationStatus::Succeeded)
.callback_details(
CallbackDetails::builder()
.callback_id("cb-1")
.result("\"ok\"")
.build(),
)
.build()
.expect("all required fields set")
}
fn declined(start: &OperationUpdate) -> Operation {
operation_like(start, OperationStatus::Failed)
.callback_details(
CallbackDetails::builder()
.callback_id("cb-1")
.error(
ErrorObject::builder()
.error_type("Declined")
.error_message("no")
.build(),
)
.build(),
)
.build()
.expect("all required fields set")
}
fn submitter_completing(
completed: impl Fn(&OperationUpdate) -> Operation + Send + Sync + 'static,
) -> ScriptedClient {
ScriptedClient::new(move |updates, history| {
let mut ops = Vec::new();
for update in updates {
if update.r#type() == &OperationType::Callback
&& update.action() == &OperationAction::Start
{
ops.push(started(update));
}
}
let submitted = updates.iter().any(|u| {
u.r#type() == &OperationType::Step && u.action() == &OperationAction::Succeed
});
if submitted
&& let Some(start) = history.iter().find(|u| {
u.r#type() == &OperationType::Callback && u.action() == &OperationAction::Start
})
{
ops.push(completed(start));
}
ops
})
}
#[tokio::test(start_paused = true)]
async fn heartbeat_timeout_reaches_the_start() {
let client = Arc::new(submitter_completing(succeeded));
let ctx = scripted_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let value = ctx
.wait_for_callback::<String>(|_step, _callback_id| async { Ok(()) })
.name("w")
.heartbeat_timeout(std::time::Duration::from_secs(90))
.await
.map_err(wire_err)?;
ctx.create_callback::<String>()
.name("cb")
.heartbeat_timeout(std::time::Duration::from_secs(90))
.await
.map_err(wire_err)?;
Ok(value)
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("ok".to_owned()));
let updates = client.updates();
let outer = named(&updates, "w", &OperationAction::Start)
.expect("the wait_for_callback started")
.id()
.to_owned();
let inner = updates
.iter()
.find(|u| {
u.r#type() == &OperationType::Callback
&& u.action() == &OperationAction::Start
&& u.parent_id() == Some(outer.as_str())
})
.expect("the inner callback started");
let created =
named(&updates, "cb", &OperationAction::Start).expect("the create_callback started");
for start in [inner, created] {
assert_eq!(start.r#type(), &OperationType::Callback, "{start:?}");
assert_eq!(
start
.callback_options()
.map(aws_sdk_lambda::types::CallbackOptions::heartbeat_timeout_seconds),
Some(90),
"{start:?}"
);
}
}
type Seen = Arc<std::sync::Mutex<Option<String>>>;
fn see_active_body(seen: &Seen) {
*seen.lock().expect("lock") = Some(format!("{:?}", crate::context::current_body()));
}
#[tokio::test(start_paused = true)]
async fn wait_for_callback_body_and_submitter() {
use crate::plugin::recorder::{UfHooks, context_over};
let hooks = UfHooks::default();
let client = Arc::new(submitter_completing(succeeded));
let ctx = context_over(
&hooks,
Vec::new(),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
);
let seen = Seen::default();
let submitter_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, move |ctx| async move {
ctx.wait_for_callback::<String>(move |_step, _callback_id| {
let seen = Arc::clone(&submitter_seen);
async move {
see_active_body(&seen);
Ok(())
}
})
.name("approve")
.await
.map_err(wire_err)
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("ok".to_owned()));
let body = hooks.user_function_of("approve", true);
assert!(
body.iter()
.all(|r| r.sub_type.as_deref() == Some(super::WFCB_SUB_TYPE)),
"{body:?}"
);
assert_eq!(hooks.find("approve", "uf_end").ok, Some(true));
let submitter = hooks.user_function_of("approve-submitter", true);
assert!(
submitter
.iter()
.all(|r| r.sub_type.as_deref() == Some("Step") && r.info_attempt == Some(1)),
"{submitter:?}"
);
assert_eq!(
hooks.find("approve-submitter", "attempt_start").attempt,
Some(1)
);
assert_eq!(hooks.find("approve-submitter", "uf_end").ok, Some(true));
let order: Vec<_> = hooks
.records()
.into_iter()
.filter(|r| r.plugin == "p0" && matches!(r.hook, "uf_start" | "uf_end"))
.map(|r| (r.hook, r.name.unwrap_or_default()))
.collect();
let at = |hook: &str, name: &str| order.iter().position(|(h, n)| *h == hook && n == name);
assert!(
at("uf_start", "approve") < at("uf_start", "approve-submitter")
&& at("uf_end", "approve-submitter") < at("uf_end", "approve"),
"{order:?}"
);
assert_eq!(
seen.lock().expect("lock").as_deref(),
Some(r#"Step("1-2")"#)
);
}
fn fail_error(update: &OperationUpdate) -> (Option<String>, Option<String>) {
crate::plugin::recorder::update_error(update)
}
#[tokio::test(start_paused = true)]
async fn wait_for_callback_failure_outcomes() {
use crate::plugin::recorder::{UfHooks, context_over, type_and_message, update_of};
let hooks = UfHooks::default();
let client = Arc::new(submitter_completing(succeeded));
let ctx = context_over(
&hooks,
Vec::new(),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
);
let seen = Seen::default();
let strategy_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, move |ctx| async move {
ctx.wait_for_callback::<String>(|_step, _callback_id| async {
Err("cannot submit".into())
})
.name("approve")
.submitter_retry(move |_err, _attempt| {
see_active_body(&strategy_seen);
crate::RetryDecision::Stop
})
.await
.map_err(wire_err)
})
.await
.expect("no panic");
assert!(
matches!(outcome, InvocationOutcome::Failed { .. }),
"{outcome:?}"
);
let updates = client.updates();
let submitter_end = hooks.find("approve-submitter", "uf_end");
let submitter_end = type_and_message(submitter_end.wire.as_ref().expect("an Err outcome"));
assert_eq!(
submitter_end,
(Some("Error".to_owned()), Some("cannot submit".to_owned()))
);
let submitter_fail = updates
.iter()
.find(|u| u.r#type() == &OperationType::Step && u.action() == &OperationAction::Fail)
.expect("the submitter's FAIL");
assert_eq!(fail_error(submitter_fail), submitter_end);
let body_end = hooks.find("approve", "uf_end");
let body_end = type_and_message(body_end.wire.as_ref().expect("an Err outcome"));
let body_fail =
update_of(&updates, "approve", &OperationAction::Fail).expect("the body's FAIL");
assert_eq!(fail_error(&body_fail), body_end);
assert_eq!(
seen.lock().expect("lock").as_deref(),
Some(r#"Context("1")"#)
);
let hooks = UfHooks::default();
let client = Arc::new(submitter_completing(declined));
let ctx = context_over(
&hooks,
Vec::new(),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
);
let outcome = drive_handler(ctx, None, |ctx| async move {
ctx.wait_for_callback::<String>(|_step, _callback_id| async { Ok(()) })
.name("approve")
.await
.map_err(wire_err)
})
.await
.expect("no panic");
assert!(
matches!(outcome, InvocationOutcome::Failed { .. }),
"{outcome:?}"
);
assert_eq!(hooks.find("approve-submitter", "uf_end").ok, Some(true));
let body_end = hooks.find("approve", "uf_end");
let body_end = type_and_message(body_end.wire.as_ref().expect("an Err outcome"));
assert_eq!(body_end.1.as_deref(), Some("no"));
let body_fail = update_of(&client.updates(), "approve", &OperationAction::Fail)
.expect("the body's FAIL");
assert_eq!(fail_error(&body_fail), body_end);
}
async fn join_callback_with_step(
client: &Arc<ScriptedClient>,
) -> (
InvocationOutcome,
Option<Result<String, crate::OperationError>>,
) {
let ctx = scripted_ctx(client);
let seen = Arc::new(std::sync::Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
*handler_seen.lock().expect("lock") = Some(result);
Ok("done".to_owned())
})
.await
.expect("no panic");
let result = seen.lock().expect("lock").take();
(outcome, result)
}
#[tokio::test(start_paused = true)]
async fn callback_resumes_on_sibling_response() {
let (events, _guard) = crate::driver::test_support::EventMessages::install();
let client = Arc::new(client_completing(succeeded));
let (outcome, result) = join_callback_with_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let value = result
.expect("the handler finished")
.expect("the callback succeeded");
assert_eq!(value, "ok");
assert_eq!(
events.replayed(),
0,
"a resumed operation emits no replay event"
);
}
#[tokio::test(start_paused = true)]
async fn resumed_callback_failure_matches_replay() {
let client = Arc::new(client_completing(declined));
let (outcome, result) = join_callback_with_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let live = result
.expect("the handler finished")
.expect_err("the callback was declined");
let start = named(&client.updates(), "cb", &OperationAction::Start)
.cloned()
.expect("the callback started");
let log = crate::client::operations_to_checkpoint_log(&[declined(&start)]);
let replay_ctx = crate::DurableContext::new_root(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
);
let replayed = replay_ctx
.create_callback::<String>()
.name("cb")
.await
.expect("the record replays")
.result()
.await
.expect_err("the replayed callback was declined");
assert_eq!(
format!("{:?}", live.kind()),
format!("{:?}", replayed.kind())
);
assert_eq!(live.wire(), replayed.wire());
}
fn not_json(start: &OperationUpdate) -> Operation {
operation_like(start, OperationStatus::Succeeded)
.callback_details(
CallbackDetails::builder()
.callback_id("cb-1")
.result("not json")
.build(),
)
.build()
.expect("all required fields set")
}
fn is_decode_failure(err: &crate::OperationError) -> bool {
matches!(
err.kind(),
crate::error::OperationErrorKind::Callback(e)
if matches!(e.kind(), crate::error::CallbackErrorKind::DeserializationFailed)
)
}
async fn live_decode_failure() -> (Arc<ScriptedClient>, crate::OperationError) {
let client = Arc::new(client_completing(not_json));
let (outcome, result) = join_callback_with_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let err = result
.expect("creation succeeded and the handler finished")
.expect_err("the payload does not decode");
assert!(is_decode_failure(&err), "{err:?}");
(client, err)
}
fn not_json_replay_ctx(client: &ScriptedClient) -> crate::DurableContext {
let start = named(&client.updates(), "cb", &OperationAction::Start)
.cloned()
.expect("the callback started");
let log = crate::client::operations_to_checkpoint_log(&[not_json(&start)]);
crate::DurableContext::new_root(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
)
}
fn assert_same_error(actual: &crate::OperationError, expected: &crate::OperationError) {
assert_eq!(
format!("{:?}", actual.kind()),
format!("{:?}", expected.kind())
);
assert_eq!(format!("{actual:#}"), format!("{expected:#}"));
assert_eq!(actual.wire(), expected.wire());
}
#[tokio::test(start_paused = true)]
async fn replayed_callback_decode_failure_comes_from_result() {
let (client, live) = live_decode_failure().await;
let ctx = not_json_replay_ctx(&client);
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.expect("creation succeeds");
let replayed = cb.result().await.expect_err("the payload does not decode");
assert_same_error(&replayed, &live);
let ctx = not_json_replay_ctx(&client);
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.expect("creation succeeds");
let spawned = cb
.spawn_result()
.await
.expect_err("the payload does not decode");
assert_same_error(&spawned, &live);
}
#[tokio::test(start_paused = true)]
async fn pending_spawn_result_decode_failure_matches_result() {
let (_, live) = live_decode_failure().await;
let client = Arc::new(client_completing(not_json));
let ctx = scripted_ctx(&client);
let seen = Arc::new(std::sync::Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.spawn_result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
*handler_seen.lock().expect("lock") = Some(result);
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let spawned = seen
.lock()
.expect("lock")
.take()
.expect("the handler finished")
.expect_err("the payload does not decode");
assert_same_error(&spawned, &live);
}
fn timed_out_with(
start: &OperationUpdate,
error_type: &str,
message: Option<&str>,
) -> Operation {
let mut error = ErrorObject::builder().error_type(error_type);
if let Some(message) = message {
error = error.error_message(message);
}
operation_like(start, OperationStatus::TimedOut)
.callback_details(
CallbackDetails::builder()
.callback_id("cb-1")
.error(error.build())
.build(),
)
.build()
.expect("all required fields set")
}
fn paused(start: &OperationUpdate) -> Operation {
operation_like(start, OperationStatus::from("PAUSED"))
.callback_details(CallbackDetails::builder().callback_id("cb-1").build())
.build()
.expect("all required fields set")
}
fn assert_unrecognized_paused(outcome: &InvocationOutcome) {
let InvocationOutcome::Failed { error } = outcome else {
unreachable!("expected a failed execution, got {outcome:?}");
};
assert_eq!(error.error_type(), Some("NonDeterministicExecutionError"));
assert!(
error.error_message().is_some_and(|m| {
m.ends_with(
"has unrecognized status \"PAUSED\": \
the SDK cannot interpret the recorded outcome",
)
}),
"{error:?}"
);
}
#[tokio::test(start_paused = true)]
async fn pending_unrecognized_status_fails_execution() {
use std::sync::atomic::{AtomicBool, Ordering};
let client = Arc::new(client_completing(paused));
let (outcome, _) = join_callback_with_step(&client).await;
assert_unrecognized_paused(&outcome);
let client = Arc::new(client_completing(paused));
let outcome = drive_handler(scripted_ctx(&client), None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.spawn_result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
Ok(format!("caught: {}", result.is_err()))
})
.await
.expect("no panic");
assert_unrecognized_paused(&outcome);
let client = Arc::new(client_completing(paused));
let outcome = drive_handler(scripted_ctx(&client), None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
ctx.step(|_| async { Ok(2_u32) })
.name("s")
.await
.map_err(wire_err)?;
Ok(format!("caught: {}", cb.result().await.is_err()))
})
.await
.expect("no panic");
assert_unrecognized_paused(&outcome);
let client = Arc::new(submitter_completing(paused));
let returned = Arc::new(AtomicBool::new(false));
let returned_h = Arc::clone(&returned);
let outcome = drive_handler(scripted_ctx(&client), None, move |ctx| async move {
let caught = ctx
.wait_for_callback::<String>(|_step, _callback_id| async { Ok(()) })
.name("w")
.await;
returned_h.store(true, Ordering::SeqCst);
Ok(format!("caught: {}", caught.is_err()))
})
.await
.expect("no panic");
assert!(
returned.load(Ordering::SeqCst),
"wait_for_callback returned its unrecognized-status error"
);
assert_unrecognized_paused(&outcome);
let written = client.updates();
assert!(
written.iter().all(|u| u.action() != &OperationAction::Fail),
"no FAIL record is written from an unrecognized status: {written:?}"
);
}
#[tokio::test(start_paused = true)]
async fn pending_timeout_message_fallback_for_result_and_spawn_result() {
let client = Arc::new(client_completing(|start| {
timed_out_with(start, "Callback.Timeout", None)
}));
let (outcome, result) = join_callback_with_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let err = result
.expect("the handler finished")
.expect_err("the callback timed out");
assert!(
matches!(
err.kind(),
crate::error::OperationErrorKind::Callback(e)
if matches!(e.kind(), crate::error::CallbackErrorKind::TimedOut)
),
"{err:?}"
);
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Callback.Timeout"));
assert_eq!(wire.error_message(), Some("Callback timed out"));
let client = Arc::new(client_completing(|start| {
timed_out_with(start, "Callback.Heartbeat", Some(""))
}));
let ctx = scripted_ctx(&client);
let seen = Arc::new(std::sync::Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.spawn_result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
*handler_seen.lock().expect("lock") = Some(result);
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let err = seen
.lock()
.expect("lock")
.take()
.expect("the handler finished")
.expect_err("the heartbeat timed out");
assert!(
matches!(
err.kind(),
crate::error::OperationErrorKind::Callback(e)
if matches!(e.kind(), crate::error::CallbackErrorKind::HeartbeatTimedOut)
),
"{err:?}"
);
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Callback.Heartbeat"));
assert_eq!(wire.error_message(), Some("Callback timed out"));
}
fn failed_with(
start: &OperationUpdate,
status: OperationStatus,
error_type: &str,
message: Option<&str>,
) -> Operation {
let mut error = ErrorObject::builder().error_type(error_type);
if let Some(message) = message {
error = error.error_message(message);
}
operation_like(start, status)
.callback_details(
CallbackDetails::builder()
.callback_id("cb-1")
.error(error.build())
.build(),
)
.build()
.expect("all required fields set")
}
#[tokio::test(start_paused = true)]
async fn pending_external_failure_message_fallback_for_result_and_spawn_result() {
let client = Arc::new(client_completing(|start| {
failed_with(start, OperationStatus::Failed, "Declined", None)
}));
let (outcome, result) = join_callback_with_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let err = result
.expect("the handler finished")
.expect_err("the callback failed");
assert!(
matches!(
err.kind(),
crate::error::OperationErrorKind::Callback(e)
if matches!(e.kind(), crate::error::CallbackErrorKind::ExternalFailure)
),
"{err:?}"
);
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Declined"));
assert_eq!(wire.error_message(), Some("Callback failed"));
let client = Arc::new(client_completing(|start| {
failed_with(start, OperationStatus::Stopped, "Halted", Some(""))
}));
let ctx = scripted_ctx(&client);
let seen = Arc::new(std::sync::Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.spawn_result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
*handler_seen.lock().expect("lock") = Some(result);
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let err = seen
.lock()
.expect("lock")
.take()
.expect("the handler finished")
.expect_err("the callback stopped");
assert!(
matches!(
err.kind(),
crate::error::OperationErrorKind::Callback(e)
if matches!(e.kind(), crate::error::CallbackErrorKind::ExternalFailure)
),
"{err:?}"
);
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Halted"));
assert_eq!(wire.error_message(), Some("Callback failed"));
}
async fn catch_decode_failure(
ctx: crate::DurableContext,
) -> Result<String, crate::error::WireError> {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let (result, step) = tokio::join!(
cb.result(),
ctx.step(|_| async { Ok(2_u32) }).name("s").future()
);
step.map_err(wire_err)?;
match result {
Err(err) if is_decode_failure(&err) => {
ctx.step(|_| async { Ok(3_u32) })
.name("after")
.await
.map_err(wire_err)?;
Ok("decode-failed".to_owned())
}
Err(err) => Err(wire_err(err)),
Ok(value) => Ok(value),
}
}
#[tokio::test(start_paused = true)]
async fn caught_callback_decode_failure_replays_the_same_path() {
let client = Arc::new(client_completing(not_json));
let outcome = drive_handler(scripted_ctx(&client), None, catch_decode_failure)
.await
.expect("no panic");
assert_eq!(
outcome,
InvocationOutcome::Complete("decode-failed".to_owned())
);
let updates = client.updates();
let mut operations = Vec::new();
for update in &updates {
if update.r#type() == &OperationType::Callback
&& update.action() == &OperationAction::Start
{
operations.push(not_json(update));
}
if update.r#type() == &OperationType::Step
&& update.action() == &OperationAction::Succeed
{
operations.push(
operation_like(update, OperationStatus::Succeeded)
.step_details(
aws_sdk_lambda::types::StepDetails::builder()
.attempt(1)
.set_result(update.payload().map(str::to_owned))
.build(),
)
.build()
.expect("all required fields set"),
);
}
}
assert_eq!(operations.len(), 3, "{updates:?}");
let log = crate::client::operations_to_checkpoint_log(&operations);
let replay_client = Arc::new(ScriptedClient::empty());
let ctx = crate::DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&replay_client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
);
let outcome = drive_handler(ctx, None, catch_decode_failure)
.await
.expect("no panic");
assert_eq!(
outcome,
InvocationOutcome::Complete("decode-failed".to_owned())
);
assert!(
replay_client.updates().is_empty(),
"the replay sends no update: {:?}",
replay_client.updates()
);
}
#[tokio::test(start_paused = true)]
async fn dropped_pending_callback_is_not_decoded() {
let decodes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let serdes = super::tests::CountingFailingSerdes(Arc::clone(&decodes));
let client = Arc::new(client_completing(not_json));
let outcome = drive_handler(scripted_ctx(&client), None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.serdes(serdes)
.await
.map_err(wire_err)?;
drop(cb);
ctx.step(|_| async { Ok(2_u32) })
.name("s")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(
client
.calls()
.iter()
.any(|(_, u)| named(u, "s", &OperationAction::Succeed).is_some()),
"step `s`'s response carried the terminal record"
);
assert_eq!(
decodes.load(std::sync::atomic::Ordering::SeqCst),
0,
"a dropped pending callback is not decoded"
);
}
#[tokio::test(start_paused = true)]
async fn spawn_result_settled_and_direct_result() {
let start = OperationUpdate::builder()
.id(crate::engine::compute_wire_id_public("1"))
.r#type(OperationType::Callback)
.sub_type(super::CALLBACK_SUB_TYPE)
.action(OperationAction::Start)
.name("cb")
.build()
.expect("all required fields set");
let log = crate::client::operations_to_checkpoint_log(&[succeeded(&start)]);
let ctx = crate::DurableContext::new_root(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
);
let tasks = Arc::clone(ctx.coordinator().tasks());
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.expect("the record replays");
let handle = cb.spawn_result();
assert_eq!(tasks.live(), 0, "a settled callback starts no task");
let value = handle.await.expect("the callback succeeded");
assert_eq!(value, "ok");
assert_eq!(tasks.live(), 0);
let client = Arc::new(crate::driver::test_support::RefreshClient::new(|req| {
let mut ops = crate::driver::test_support::callback_starts(req.updates);
if req.empty_number == 1
&& let Some(start) = named(req.history, "cb", &OperationAction::Start)
{
ops.push(succeeded(start));
}
Ok(ops)
}));
let ctx = crate::driver::test_support::refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let step = ctx
.step(|_| async {
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
Ok(1_u32)
})
.name("s")
.future();
let (result, step) = tokio::join!(cb.result(), step);
step.map_err(wire_err)?;
result.map_err(wire_err)
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("ok".to_owned()));
crate::driver::test_support::assert_times(
&client.empty_calls(),
&[std::time::Duration::from_secs(1)],
);
}
#[tokio::test]
async fn spawn_result_outside_runtime_returns_error() {
let client = Arc::new(ScriptedClient::new(|updates, _| {
updates
.iter()
.filter(|u| u.r#type() == &OperationType::Callback)
.map(started)
.collect()
}));
let ctx = scripted_ctx(&client);
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.expect("the callback starts");
let polled =
crate::task::test_support::poll_once_without_runtime(move || cb.spawn_result());
let std::task::Poll::Ready(Err(err)) = polled else {
unreachable!("expected an immediate error, got {polled:?}");
};
assert!(
format!("{err:#}").contains("does not own the context"),
"unexpected error: {err:#}"
);
let fatal = ctx
.coordinator()
.fatal_error()
.expect("a fatal error is recorded");
assert!(
fatal
.error_message
.starts_with("context usage error in \"spawn\""),
"unexpected fatal error: {fatal:?}"
);
}
fn submitter<'a>(
updates: &'a [OperationUpdate],
action: &OperationAction,
) -> Option<&'a OperationUpdate> {
updates.iter().find(|u| {
u.r#type() == &OperationType::Step && u.name().is_none() && u.action() == action
})
}
fn submitter_record(
req: &crate::driver::test_support::RefreshRequest<'_>,
update: &OperationUpdate,
status: OperationStatus,
next_in: Option<std::time::Duration>,
) -> Operation {
let mut details = aws_sdk_lambda::types::StepDetails::builder().attempt(1);
if let Some(next_in) = next_in {
details = details.next_attempt_timestamp(req.timestamp_in(next_in));
}
operation_like(update, status)
.step_details(details.build())
.build()
.expect("all required fields set")
}
#[tokio::test(start_paused = true)]
async fn submitter_retry_runs_in_invocation() {
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Duration;
let client = Arc::new(crate::driver::test_support::RefreshClient::new(|req| {
let mut ops = crate::driver::test_support::callback_starts(req.updates);
if let Some(retry) = submitter(req.updates, &OperationAction::Retry) {
ops.push(submitter_record(
req,
retry,
OperationStatus::Pending,
Some(Duration::from_secs(1)),
));
}
if req.empty_number == 1
&& let Some(retry) = submitter(req.history, &OperationAction::Retry)
{
ops.push(submitter_record(req, retry, OperationStatus::Ready, None));
}
if req.empty_number >= 2
&& let Some(start) = req.history.iter().find(|u| {
u.r#type() == &OperationType::Callback && u.action() == &OperationAction::Start
})
{
ops.push(succeeded(start));
}
Ok(ops)
}));
let ctx = crate::driver::test_support::refresh_ctx(&client);
let runs = Arc::new(AtomicU32::new(0));
let submitter_runs = Arc::clone(&runs);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let wfcb = ctx
.wait_for_callback::<String>(move |_, _callback_id| {
let runs = Arc::clone(&submitter_runs);
async move {
if runs.fetch_add(1, Ordering::SeqCst) == 0 {
return Err("transient".into());
}
Ok(())
}
})
.name("wfcb")
.submitter_retry(|_err, attempt| {
if attempt < 2 {
crate::RetryDecision::retry(Duration::from_secs(1))
} else {
crate::RetryDecision::Stop
}
})
.future();
let step = ctx
.step(|_| async {
tokio::time::sleep(Duration::from_secs(3)).await;
Ok(1_u32)
})
.name("s")
.future();
let (result, step) = tokio::join!(wfcb, step);
step.map_err(wire_err)?;
result.map_err(wire_err)
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("ok".to_owned()));
assert_eq!(runs.load(Ordering::SeqCst), 2, "the submitter ran twice");
}
fn refresh_completing(
completed: impl Fn(&OperationUpdate) -> Operation + Send + Sync + 'static,
) -> crate::driver::test_support::RefreshClient {
crate::driver::test_support::RefreshClient::new(move |req| {
if let Some(start) = named(req.updates, "cb", &OperationAction::Start) {
return Ok(vec![started(start)]);
}
if req.empty_number == 1
&& let Some(start) = named(req.history, "cb", &OperationAction::Start)
{
return Ok(vec![completed(start)]);
}
Ok(Vec::new())
})
}
async fn join_result_with_sleeping_step(
client: crate::driver::test_support::RefreshClient,
hooks: &crate::plugin::test_support::OpHooks,
) -> (
InvocationOutcome,
Option<(Result<String, crate::OperationError>, usize)>,
) {
let ctx = crate::DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::new(client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
std::time::Duration::ZERO,
0,
Some(hooks.plugins()),
);
let seen = Arc::new(std::sync::Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let handler_hooks = hooks.clone();
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let step = ctx
.step(|_| async {
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
Ok(1_u32)
})
.name("s")
.spawn();
let result = async {
let result = cb.result().await;
(result, handler_hooks.of("cb").len())
};
let (result, step) = tokio::join!(result, step);
step.map_err(wire_err)?;
*handler_seen.lock().expect("lock") = Some(result);
Ok("done".to_owned())
})
.await
.expect("no panic");
let result = seen.lock().expect("lock").take();
(outcome, result)
}
#[tokio::test(start_paused = true)]
async fn callback_resumed_in_invocation_ends_live() {
let hooks = crate::plugin::test_support::OpHooks::default();
let (outcome, result) =
join_result_with_sleeping_step(refresh_completing(succeeded), &hooks).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let (value, hooks_at_return) = result.expect("the handler finished");
assert_eq!(value.expect("the callback succeeded"), "ok");
let shapes: Vec<_> = hooks
.of("cb")
.iter()
.map(|r| (r.hook, r.replay, r.status))
.collect();
assert_eq!(
shapes,
[
("start", false, Some(crate::OperationStatus::Started)),
("end", false, Some(crate::OperationStatus::Succeeded)),
]
);
assert_eq!(
hooks_at_return, 2,
"the end hook ran before result() returned"
);
}
#[tokio::test(start_paused = true)]
async fn callback_failed_in_invocation_ends_live() {
let hooks = crate::plugin::test_support::OpHooks::default();
let (outcome, result) =
join_result_with_sleeping_step(refresh_completing(declined), &hooks).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let (value, _) = result.expect("the handler finished");
value.expect_err("the callback was declined");
let shapes: Vec<_> = hooks
.of("cb")
.iter()
.map(|r| (r.hook, r.replay, r.status))
.collect();
assert_eq!(
shapes,
[
("start", false, Some(crate::OperationStatus::Started)),
("end", false, Some(crate::OperationStatus::Failed)),
]
);
}
#[tokio::test(start_paused = true)]
async fn callback_failure_statuses_in_invocation_end_live() {
type Fields = Option<(Option<&'static str>, Option<&'static str>, bool)>;
let statuses = [
(OperationStatus::Failed, crate::OperationStatus::Failed),
(
OperationStatus::Cancelled,
crate::OperationStatus::Cancelled,
),
(OperationStatus::Stopped, crate::OperationStatus::Stopped),
(OperationStatus::TimedOut, crate::OperationStatus::TimedOut),
];
let cases: [Fields; 3] = [
Some((Some("Declined"), Some("no"), true)),
Some((Some("Declined"), None, false)),
None,
];
for (status, expected_status) in statuses {
for fields in cases {
let hooks = crate::plugin::test_support::OpHooks::default();
let status_for = status.clone();
let completed = move |start: &OperationUpdate| {
let mut details = CallbackDetails::builder().callback_id("cb-1");
if let Some((error_type, message, full)) = fields {
let mut error = ErrorObject::builder();
if let Some(error_type) = error_type {
error = error.error_type(error_type);
}
if let Some(message) = message {
error = error.error_message(message);
}
if full {
error = error
.error_data("d")
.stack_trace("frame-1")
.stack_trace("frame-2");
}
details = details.error(error.build());
}
operation_like(start, status_for.clone())
.callback_details(details.build())
.build()
.expect("all required fields set")
};
let (outcome, result) =
join_result_with_sleeping_step(refresh_completing(completed), &hooks).await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let (value, hooks_at_return) = result.expect("the handler finished");
let err = value.expect_err("the callback did not succeed");
let records = hooks.of("cb");
let shapes: Vec<_> = records
.iter()
.map(|r| (r.hook, r.replay, r.status))
.collect();
assert_eq!(
shapes,
[
("start", false, Some(crate::OperationStatus::Started)),
("end", false, Some(expected_status)),
],
"{status:?} {fields:?}"
);
assert_eq!(
hooks_at_return, 2,
"the end hook ran before result() returned"
);
let end = records.last().expect("the end hook");
if fields.is_some() {
assert_eq!(end.info_error.as_ref(), err.wire(), "{status:?} {fields:?}");
} else {
assert_eq!(end.info_error, None, "{status:?}");
}
let message = err.wire().and_then(crate::WireError::error_message);
let expected_message = match (fields, &status) {
(Some((_, Some(message), _)), _) => message,
(_, OperationStatus::TimedOut) => "Callback timed out",
_ => "Callback failed",
};
assert_eq!(message, Some(expected_message), "{status:?} {fields:?}");
if let Some((_, _, true)) = fields {
let wire = err.wire().expect("the record travels on the error");
assert_eq!(wire.error_type(), Some("Declined"));
assert_eq!(wire.error_data(), Some("d"));
assert_eq!(wire.stack_trace(), ["frame-1", "frame-2"]);
}
}
}
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod start_order_tests {
use std::future::{Future, IntoFuture as _};
use std::sync::Arc;
use std::time::Duration;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use super::WfcbAfter;
use crate::client::InMemoryExecutionClient;
use crate::context::DurableContext;
use crate::engine::CheckpointLog;
use crate::error::{CallbackError, CallbackErrorKind, OperationError, OperationErrorKind};
type Submission = (String, String, OperationAction);
fn recorded_context(
client: Arc<dyn crate::client::ExecutionClient>,
submissions: &Arc<std::sync::Mutex<Vec<Submission>>>,
hooks: Option<&crate::plugin::test_support::OpHooks>,
) -> DurableContext {
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(Vec::new())),
client,
"token-0".to_owned(),
Duration::ZERO,
0,
hooks.map(crate::plugin::test_support::OpHooks::plugins),
);
let recorded = Arc::clone(submissions);
ctx.set_submission_recorder(Arc::new(move |updates, _flag| {
let mut recorded = recorded.lock().unwrap();
for u in updates {
recorded.push((
u.name().unwrap_or("-").to_owned(),
u.id().to_owned(),
u.action().clone(),
));
}
}));
ctx
}
async fn wait_until(what: &str, mut ready: impl FnMut() -> bool) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
while !ready() {
assert!(tokio::time::Instant::now() < deadline, "timed out: {what}");
tokio::time::sleep(Duration::from_millis(1)).await;
}
}
async fn wfcb_outcome_waits_for_failed_start(
body_result: Result<String, OperationError>,
action: OperationAction,
) {
let submissions: Arc<std::sync::Mutex<Vec<Submission>>> = Arc::default();
let ctx = recorded_context(
Arc::new(InMemoryExecutionClient::new(Vec::new())),
&submissions,
None,
);
let entries: Arc<std::sync::Mutex<Vec<(String, OperationAction)>>> = Arc::default();
let entered = Arc::clone(&entries);
ctx.set_settler_entry_recorder(Arc::new(move |wire_id, action| {
entered.lock().unwrap().push((wire_id.to_owned(), action));
}));
let wire_id = crate::engine::compute_wire_id_public("1");
let (sender, receiver) = tokio::sync::oneshot::channel();
let after = WfcbAfter {
ctx: ctx.clone(),
wire_id: wire_id.clone(),
name: Some("wcb".to_owned()),
};
let settle = after.settle(
Some(crate::context::SubmittedWrite::from_receiver(receiver)),
crate::plugin::Ran::from_result(body_result, crate::callback::extract_wire_error),
);
let outcomes = |submissions: &[Submission]| {
submissions
.iter()
.filter(|(_, id, action)| {
*id == wire_id
&& matches!(action, OperationAction::Succeed | OperationAction::Fail)
})
.count()
};
let observe = async {
wait_until("the settler-entry marker", || {
entries
.lock()
.unwrap()
.contains(&(wire_id.clone(), action.clone()))
})
.await;
assert_eq!(outcomes(&submissions.lock().unwrap()), 0, "no outcome yet");
let rejection = crate::client::ExecutionClientError::new(
crate::client::ExecutionClientErrorScope::Execution,
"START rejected",
);
sender.send(Err(rejection)).expect("the settler waits");
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(outcomes(&submissions.lock().unwrap()), 0, "no outcome");
assert!(ctx.coordinator().fatal_error().is_none());
};
tokio::select! {
_ = settle => panic!("the settler resolved after a failed START"),
() = observe => {}
}
}
#[tokio::test]
async fn wait_for_callback_success_waits_for_failed_start() {
wfcb_outcome_waits_for_failed_start(Ok("ok".to_owned()), OperationAction::Succeed).await;
}
#[tokio::test]
async fn wait_for_callback_failure_waits_for_failed_start() {
let failed = Err(OperationError::from_kind(OperationErrorKind::Callback(
CallbackError::new(CallbackErrorKind::ExternalFailure, Some("declined".into())),
)));
wfcb_outcome_waits_for_failed_start(failed, OperationAction::Fail).await;
}
struct ResponseLog(Arc<std::sync::Mutex<Vec<String>>>);
impl crate::client::ExecutionClient for ResponseLog {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> std::pin::Pin<
Box<
dyn Future<
Output = Result<
crate::client::CheckpointOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
let mut records = Vec::new();
{
let mut log = self.0.lock().unwrap();
for u in updates
.iter()
.filter(|u| u.action() == &OperationAction::Start)
{
log.push(format!("response {}", u.name().unwrap_or("-")));
if u.r#type() == &OperationType::Callback {
records.push(
aws_sdk_lambda::types::Operation::builder()
.id(u.id())
.r#type(OperationType::Callback)
.status(aws_sdk_lambda::types::OperationStatus::Started)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.callback_details(
aws_sdk_lambda::types::CallbackDetails::builder()
.callback_id("cb-id")
.build(),
)
.build()
.expect("every required Operation field is set"),
);
}
}
}
Box::pin(async move {
Ok(crate::client::CheckpointOutput::new(
Some("t".to_owned()),
records,
None,
))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> std::pin::Pin<
Box<
dyn Future<
Output = Result<
crate::client::GetStateOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
Box::pin(async { Ok(crate::client::GetStateOutput::new(Vec::new(), None)) })
}
}
#[tokio::test]
async fn wait_for_callback_body_starts_before_start_response() {
let log: Arc<std::sync::Mutex<Vec<String>>> = Arc::default();
let probe_log = Arc::clone(&log);
let hooks = crate::plugin::test_support::OpHooks::probing(move || {
probe_log.lock().unwrap().len().to_string()
});
let submissions: Arc<std::sync::Mutex<Vec<Submission>>> = Arc::default();
let ctx = recorded_context(
Arc::new(ResponseLog(Arc::clone(&log))),
&submissions,
Some(&hooks),
);
let marker_log = Arc::clone(&log);
let recorded = Arc::clone(&submissions);
ctx.set_submission_recorder(Arc::new(move |updates, _flag| {
for u in updates {
if u.action() == &OperationAction::Start && u.r#type() == &OperationType::Callback {
marker_log
.lock()
.unwrap()
.push("inner callback START submitted".to_owned());
}
recorded.lock().unwrap().push((
u.name().unwrap_or("-").to_owned(),
u.id().to_owned(),
u.action().clone(),
));
}
}));
let gate = Arc::new(tokio::sync::Notify::new());
let held = Arc::clone(&gate);
ctx.set_pre_lock_gate(Arc::new(move |updates: &[OperationUpdate]| {
updates
.iter()
.any(|u| u.action() == &OperationAction::Start && u.name() == Some("wcb"))
.then(|| Arc::clone(&held))
}));
let operation = ctx
.wait_for_callback::<String>(|_, _id| async { Ok(()) })
.name("wcb")
.into_future();
let marker = "inner callback START submitted";
let observe = async {
wait_until("the inner callback START", || {
log.lock().unwrap().iter().any(|e| e == marker)
})
.await;
assert!(
!log.lock().unwrap().iter().any(|e| e == "response wcb"),
"the body ran while the outer START was held: {:?}",
log.lock().unwrap()
);
gate.notify_one();
wait_until("the outer START response", || {
log.lock().unwrap().iter().any(|e| e == "response wcb")
})
.await;
};
tokio::select! {
_ = operation => {}
() = observe => {}
}
let entries = log.lock().unwrap().clone();
let index = |entry: &str| {
entries
.iter()
.position(|e| e == entry)
.unwrap_or_else(|| panic!("{entry:?} is missing from {entries:?}"))
};
let hook: usize = hooks
.of("wcb")
.into_iter()
.find(|r| r.hook == "start")
.expect("the start hook of wcb fired")
.probe
.parse()
.expect("the probe is a log length");
assert!(
hook <= index(marker) && index(marker) < index("response wcb"),
"hook at {hook}: {entries:?}"
);
}
}