use std::future::Future;
use std::sync::Arc;
use serde::Serialize;
use serde::de::DeserializeOwned;
use crate::child::CHECKPOINT_SIZE_LIMIT_BYTES;
use crate::context::{DurableContext, SubmittedWrite};
use crate::engine::{CheckpointStatus, OperationId};
use crate::error::{
ChildContextError, ChildContextErrorKind, CombinatorError, CombinatorErrorKind, OperationError,
OperationErrorKind,
};
use crate::future::DurableFuture;
use crate::task::{NoRuntime, TrackedJoinSet};
pub(crate) const COMBINATOR_SUB_TYPE: &str = "RunInChildContext";
const COMBINATOR_ERROR_TYPE: &str = "CombinatorError";
const FIRST_SETTLED_FAILED_ERROR_TYPE: &str = "CombinatorError.FirstSettledFailed";
const EMPTY_INPUT_ERROR_TYPE: &str = "CombinatorError.EmptyInput";
const JOIN_FAILED_ERROR_TYPE_PREFIX: &str = "CombinatorError.JoinFailed.";
const ALL_FAILED_ERROR_TYPE: &str = "CombinatorError.AllFailed";
#[derive(serde::Serialize, serde::Deserialize)]
struct SerializedLoser {
#[serde(default, skip_serializing_if = "Option::is_none")]
error_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_data: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
stack_trace: Vec<String>,
}
impl SerializedLoser {
fn from_wire(wire: &crate::error::WireError) -> Self {
Self {
error_type: wire.error_type().map(str::to_owned),
message: wire.error_message().map(str::to_owned),
error_data: wire.error_data().map(str::to_owned),
stack_trace: wire.stack_trace().to_vec(),
}
}
fn into_wire(self) -> crate::error::WireError {
crate::error::WireError::new(self.error_type, self.message)
.with_error_data(self.error_data)
.with_stack_trace(self.stack_trace)
}
}
#[derive(serde::Serialize, serde::Deserialize)]
struct RecordedWinner {
winner: usize,
}
pub(crate) struct TryJoinAllExecution<O> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) futures: Vec<DurableFuture<O>>,
}
impl<O: Serialize + DeserializeOwned + Send + 'static> TryJoinAllExecution<O> {
#[expect(clippy::too_many_lines)] pub(crate) async fn execute(self) -> Result<Vec<O>, OperationError> {
self.ctx.enforce_task_ownership()?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let run = if let Some(view) = self.ctx.checkpoint_view_validated(
&positional_id,
&wire_id,
"Context",
Some(COMBINATOR_SUB_TYPE),
self.name.as_deref(),
)? {
match view.status {
CheckpointStatus::Succeeded if view.replay_children => Run::Reconstruct,
CheckpointStatus::Succeeded => {
let payload = self.ctx.checkpoint_result_payload(&positional_id);
let value = replay_vec_success::<O>(payload.as_ref())?;
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
return Ok(value);
}
CheckpointStatus::Failed => {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
let wire = self
.ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Err(replay_combinator_failure(wire, &wire_id));
}
_ => Run::Resumed,
}
} else {
Run::Live
};
let reconstruct = run == Run::Reconstruct;
let count = self.futures.len();
let mut results: Vec<Option<O>> = (0..count).map(|_| None).collect();
let Running {
mut join_set,
start,
} = start_inputs(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
self.futures,
)
.await?;
let join = async move {
while let Some((_, joined)) = join_set.join_next_with_id().await {
match joined.into_value() {
(idx, Ok(value)) => {
if let Some(slot) = results.get_mut(idx) {
*slot = Some(value);
}
}
(idx, Err(op_err)) => {
join_set.abort_all();
return Err((idx, op_err));
}
}
}
Ok(results)
};
let joined = run_join(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
join,
join_failed_wire,
)
.await;
let results = match joined {
Ok(results) => results,
Err(((failed_index, op_err), wire)) => {
if !reconstruct {
checkpoint_fail(&self.ctx, &wire_id, self.name.as_deref(), start, &wire)
.await?;
}
return Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::JoinFailed(crate::error::JoinFailed::new(
failed_index,
)),
vec![Box::new(op_err)],
),
))
.with_operation(&wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire));
}
};
let mut collected: Vec<O> = Vec::with_capacity(count);
for (idx, opt) in results.into_iter().enumerate() {
match opt {
Some(v) => collected.push(v),
None => {
return Err(combinator_internal_error(&format!(
"internal: slot {idx} not filled despite no error"
)));
}
}
}
let serialized = serialize_result(&collected)?;
if !reconstruct {
checkpoint_collection_succeed(
&self.ctx,
&wire_id,
self.name.as_deref(),
start,
&serialized,
)
.await?;
}
drop(collected);
replay_vec_success::<O>(Some(&serialized))
}
}
pub(crate) struct JoinAllExecution<O> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) futures: Vec<DurableFuture<O>>,
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(tag = "status")]
enum SerializedOutcome<O> {
#[serde(rename = "fulfilled")]
Fulfilled { value: O },
#[serde(rename = "rejected")]
Rejected {
message: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_data: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
stack_trace: Vec<String>,
},
}
impl<O: Serialize + DeserializeOwned + Send + 'static> JoinAllExecution<O> {
#[expect(clippy::too_many_lines)] pub(crate) async fn execute(self) -> Result<Vec<Result<O, OperationError>>, OperationError> {
self.ctx.enforce_task_ownership()?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let run = if let Some(view) = self.ctx.checkpoint_view_validated(
&positional_id,
&wire_id,
"Context",
Some(COMBINATOR_SUB_TYPE),
self.name.as_deref(),
)? {
match view.status {
CheckpointStatus::Succeeded if view.replay_children => Run::Reconstruct,
CheckpointStatus::Succeeded => {
let payload = self.ctx.checkpoint_result_payload(&positional_id);
let value = replay_outcomes_success::<O>(payload.as_ref())?;
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
return Ok(value);
}
CheckpointStatus::Failed => {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
let wire = self
.ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Err(replay_combinator_failure(wire, &wire_id));
}
_ => Run::Resumed,
}
} else {
Run::Live
};
let reconstruct = run == Run::Reconstruct;
let count = self.futures.len();
let mut outcomes: Vec<Option<Result<O, OperationError>>> =
(0..count).map(|_| None).collect();
let Running {
mut join_set,
start,
} = start_inputs(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
self.futures,
)
.await?;
let join = async move {
while let Some((_, joined)) = join_set.join_next_with_id().await {
let (idx, outcome) = joined.into_value();
if let Some(slot) = outcomes.get_mut(idx) {
*slot = Some(outcome);
}
}
Ok(outcomes)
};
let outcomes = match run_join(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
join,
crate::plugin::never_fails,
)
.await
{
Ok(outcomes) => outcomes,
Err((never, _)) => match never {},
};
let collected: Vec<Result<O, OperationError>> = outcomes
.into_iter()
.map(|opt| {
opt.unwrap_or_else(|| Err(combinator_internal_error("future did not complete")))
})
.collect();
let serialized = {
let serializable: Vec<SerializedOutcome<&O>> = collected
.iter()
.map(|s| match s {
Ok(v) => SerializedOutcome::Fulfilled { value: v },
Err(err) => {
let wire = err.wire().cloned().unwrap_or_else(|| {
crate::error::wire_error_for(err, COMBINATOR_ERROR_TYPE)
});
SerializedOutcome::Rejected {
message: wire
.error_message()
.map_or_else(|| crate::error::chain_string(err), str::to_owned),
error_type: wire.error_type().map(str::to_owned),
error_data: wire.error_data().map(str::to_owned),
stack_trace: wire.stack_trace().to_vec(),
}
}
})
.collect();
serialize_result(&serializable)?
};
if !reconstruct {
checkpoint_collection_succeed(
&self.ctx,
&wire_id,
self.name.as_deref(),
start,
&serialized,
)
.await?;
}
drop(collected);
replay_outcomes_success::<O>(Some(&serialized))
}
}
pub(crate) struct SelectOkExecution<O> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) futures: Vec<DurableFuture<O>>,
}
impl<O: Serialize + DeserializeOwned + Send + 'static> SelectOkExecution<O> {
pub(crate) async fn execute(self) -> Result<O, OperationError> {
self.ctx.enforce_task_ownership()?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let run = if let Some(view) = self.ctx.checkpoint_view_validated(
&positional_id,
&wire_id,
"Context",
Some(COMBINATOR_SUB_TYPE),
self.name.as_deref(),
)? {
match view.status {
CheckpointStatus::Succeeded if view.replay_children => {
return replay_recorded_winner(&self.ctx, &positional_id, self.futures).await;
}
CheckpointStatus::Succeeded => {
let payload = self.ctx.checkpoint_result_payload(&positional_id);
let value = replay_single_success::<O>(payload.as_ref())?;
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
return Ok(value);
}
CheckpointStatus::Failed => {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
let wire = self
.ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Err(replay_combinator_failure(wire, &wire_id));
}
_ => Run::Resumed,
}
} else {
Run::Live
};
let empty = self.futures.is_empty();
let Running {
mut join_set,
start,
} = start_inputs(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
self.futures,
)
.await?;
let join = async move {
if empty {
return Err(FirstFailure::Empty("select_ok called with no futures"));
}
let mut errors: Vec<(usize, crate::error::Source)> = Vec::new();
while let Some((_, joined)) = join_set.join_next_with_id().await {
match joined.into_value() {
(idx, Ok(value)) => {
join_set.abort_all();
return Ok((idx, value));
}
(idx, Err(op_err)) => {
errors.push((idx, Box::new(op_err)));
}
}
}
errors.sort_by_key(|(idx, _)| *idx);
Err(FirstFailure::AllFailed(
errors.into_iter().map(|(_, err)| err).collect(),
))
};
let joined = run_join(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
join,
first_failure_wire,
)
.await;
settle_first(&self.ctx, &wire_id, self.name.as_deref(), start, joined).await
}
}
async fn settle_first<O: Serialize + DeserializeOwned + Send + 'static>(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
start: Option<SubmittedWrite>,
joined: Result<(usize, O), (FirstFailure, crate::error::WireError)>,
) -> Result<O, OperationError> {
match joined {
Ok((idx, value)) => {
let serialized = serialize_result(&value)?;
checkpoint_winner_succeed(ctx, wire_id, name, start, idx, &serialized).await?;
drop(value);
replay_single_success::<O>(Some(&serialized))
}
Err((failure, wire)) => {
checkpoint_fail(ctx, wire_id, name, start, &wire).await?;
let error = match failure {
FirstFailure::Empty(_) => combinator_empty_input_error(),
FirstFailure::AllFailed(losers) => {
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::AllFailed,
losers,
)))
}
FirstFailure::Settled(op_err) => {
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::FirstSettledFailed,
vec![Box::new(op_err)],
)))
}
};
Err(error
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire))
}
}
}
pub(crate) struct RaceExecution<O> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) futures: Vec<DurableFuture<O>>,
}
impl<O: Serialize + DeserializeOwned + Send + 'static> RaceExecution<O> {
pub(crate) async fn execute(self) -> Result<O, OperationError> {
self.ctx.enforce_task_ownership()?;
let positional_id = self.op_id.positional().to_owned();
let wire_id = self.op_id.wire().to_owned();
let run = if let Some(view) = self.ctx.checkpoint_view_validated(
&positional_id,
&wire_id,
"Context",
Some(COMBINATOR_SUB_TYPE),
self.name.as_deref(),
)? {
match view.status {
CheckpointStatus::Succeeded if view.replay_children => {
return replay_recorded_winner(&self.ctx, &positional_id, self.futures).await;
}
CheckpointStatus::Succeeded => {
let payload = self.ctx.checkpoint_result_payload(&positional_id);
let value = replay_single_success::<O>(payload.as_ref())?;
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
return Ok(value);
}
CheckpointStatus::Failed => {
self.ctx.emit_operation_replayed(
&wire_id,
self.name.as_deref(),
"Context",
Some(COMBINATOR_SUB_TYPE),
view.attempt,
);
let wire = self
.ctx
.checkpoint_wire_error(&positional_id)
.unwrap_or_default();
return Err(replay_combinator_failure(wire, &wire_id));
}
_ => Run::Resumed,
}
} else {
Run::Live
};
let empty = self.futures.is_empty();
let Running {
mut join_set,
start,
} = start_inputs(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
self.futures,
)
.await?;
let join = async move {
if empty {
return Err(FirstFailure::Empty("race called with no futures"));
}
#[expect(clippy::expect_used)]
let (_, joined) = join_set
.join_next_with_id()
.await
.expect("a non-empty race joins an input");
let (idx, settled_result) = joined.into_value();
join_set.abort_all();
settled_result
.map(|value| (idx, value))
.map_err(FirstFailure::Settled)
};
let joined = run_join(
&self.ctx,
&positional_id,
&wire_id,
self.name.as_deref(),
run,
join,
first_failure_wire,
)
.await;
settle_first(&self.ctx, &wire_id, self.name.as_deref(), start, joined).await
}
}
#[cfg(test)]
type RemainderRecorder = Box<dyn Fn(usize)>;
#[cfg(test)]
thread_local! {
static REMAINDER_RECORDER: std::cell::RefCell<Option<RemainderRecorder>> =
const { std::cell::RefCell::new(None) };
}
#[cfg(test)]
fn record_remainder(idx: usize) {
REMAINDER_RECORDER.with(|recorder| {
if let Some(recorder) = &*recorder.borrow() {
recorder(idx);
}
});
}
fn spawn_constituent<O: Send + 'static>(
ctx: &DurableContext,
join_set: &mut TrackedJoinSet<(usize, Result<O, OperationError>)>,
idx: usize,
future: DurableFuture<O>,
) -> Result<(), OperationError> {
join_set
.spawn(future.task_context(), ctx.task_ownership(), async move {
(idx, future.await)
})
.map(drop)
.map_err(NoRuntime::operation_error)
}
fn unclaimed_input_message<O: Send + 'static>(
outer: &str,
futures: &[DurableFuture<O>],
) -> Option<String> {
futures
.iter()
.find(|future| future.is_unclaimed_combinator())
.and_then(|future| future.unclaimed_input_message(outer))
}
fn reject_unclaimed(ctx: &DurableContext, message: &str) -> OperationError {
ctx.coordinator()
.record_fatal("Error".to_owned(), message.to_owned());
combinator_internal_error(message)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Run {
Live,
Resumed,
Reconstruct,
}
struct Running<O> {
join_set: TrackedJoinSet<(usize, Result<O, OperationError>)>,
start: Option<SubmittedWrite>,
}
async fn start_inputs<O: Send + 'static>(
ctx: &DurableContext,
positional_id: &str,
wire_id: &str,
name: Option<&str>,
run: Run,
futures: Vec<DurableFuture<O>>,
) -> Result<Running<O>, OperationError> {
if let Some(message) = unclaimed_input_message(positional_id, &futures) {
drop(futures);
if let Some(start) = combinator_start(ctx, wire_id, name, run)
&& let Err(err) = start.await
{
return ctx.checkpoint_failure_unrecoverable(err).await;
}
return Err(reject_unclaimed(ctx, &message));
}
let mut join_set = constituent_set(ctx);
let mut remainders = Vec::with_capacity(futures.len());
for (idx, mut future) in futures.into_iter().enumerate() {
if future.starts_on_first_poll() {
let (polled, acknowledged) = tokio::sync::oneshot::channel();
let task_cx = future.task_context();
let future = FirstPoll {
future,
polled: Some(polled),
};
join_set
.spawn(
task_cx,
ctx.task_ownership(),
async move { (idx, future.await) },
)
.map_err(NoRuntime::operation_error)?;
let _polled = acknowledged.await;
} else {
future.begin_in_creator_context();
remainders.push((idx, future));
}
}
let start = combinator_start(ctx, wire_id, name, run);
for (idx, future) in remainders {
#[cfg(test)]
record_remainder(idx);
spawn_constituent(ctx, &mut join_set, idx, future)?;
}
Ok(Running { join_set, start })
}
struct FirstPoll<O> {
future: DurableFuture<O>,
polled: Option<tokio::sync::oneshot::Sender<()>>,
}
impl<O: Send + 'static> Future for FirstPoll<O> {
type Output = Result<O, OperationError>;
fn poll(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let this = &mut *self;
let poll = std::pin::Pin::new(&mut this.future).poll(cx);
if let Some(polled) = this.polled.take() {
let _sent = polled.send(());
}
poll
}
}
fn combinator_start(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
run: Run,
) -> Option<SubmittedWrite> {
match run {
Run::Live => Some(checkpoint_start(ctx, wire_id, name)),
Run::Resumed => {
combinator_resumed(ctx, wire_id, name);
None
}
Run::Reconstruct => None,
}
}
fn constituent_set<O: Send + 'static>(
ctx: &DurableContext,
) -> TrackedJoinSet<(usize, Result<O, OperationError>)> {
TrackedJoinSet::new(Arc::clone(ctx.coordinator()))
}
fn fatal_gate(ctx: &DurableContext) -> Result<(), OperationError> {
if let Some(fatal) = ctx.coordinator().fatal_error() {
return Err(combinator_internal_error(&fatal.error_message));
}
Ok(())
}
fn checkpoint_start(ctx: &DurableContext, wire_id: &str, name: Option<&str>) -> SubmittedWrite {
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Context)
.sub_type(COMBINATOR_SUB_TYPE.to_owned())
.action(OperationAction::Start);
if let Some(n) = name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
let reserved = ctx.reserve_checkpoint_updates(vec![update]);
if let Some(plugins) = ctx.plugins() {
let info = combinator_info(ctx, wire_id, name)
.with_status(crate::OperationStatus::Started)
.with_replay(false);
crate::plugin::operation_started(plugins, &info);
}
reserved.release()
}
fn combinator_info(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
) -> crate::plugin::OperationInfo {
crate::plugin::OperationInfo::for_operation(
ctx,
wire_id,
crate::OperationKind::Context,
COMBINATOR_SUB_TYPE,
name,
)
}
async fn run_join<T, E, B>(
ctx: &DurableContext,
positional_id: &str,
wire_id: &str,
name: Option<&str>,
run: Run,
body: B,
wire: fn(&E) -> crate::error::WireError,
) -> Result<T, (E, crate::error::WireError)>
where
B: Future<Output = Result<T, E>>,
{
if run == Run::Reconstruct {
return body.await.map_err(|error| {
let derived = wire(&error);
(error, derived)
});
}
let info = combinator_info(ctx, wire_id, name);
let info = if run == Run::Live {
info.with_status(crate::OperationStatus::Started)
.with_replay(false)
} else {
info.with_replay(true)
};
crate::plugin::run_user_function_at(
ctx.shared_plugins(),
Arc::new(info),
crate::context::current_body(),
positional_id.to_owned(),
tracing::Span::current(),
move || body,
wire,
)
.await
.into_parts()
}
fn join_failed_wire((index, error): &(usize, OperationError)) -> crate::error::WireError {
crate::error::wire_error_with_type(error, &format!("{JOIN_FAILED_ERROR_TYPE_PREFIX}{index}"))
}
enum FirstFailure {
Empty(&'static str),
AllFailed(Vec<crate::error::Source>),
Settled(OperationError),
}
fn first_failure_wire(failure: &FirstFailure) -> crate::error::WireError {
match failure {
FirstFailure::Empty(message) => {
crate::error::WireError::new(Some(EMPTY_INPUT_ERROR_TYPE), Some(*message))
}
FirstFailure::AllFailed(losers) => all_failed_wire(losers),
FirstFailure::Settled(error) => {
crate::error::wire_error_with_type(error, FIRST_SETTLED_FAILED_ERROR_TYPE)
}
}
}
fn all_failed_wire(losers: &[crate::error::Source]) -> crate::error::WireError {
let loser_summary = losers
.iter()
.enumerate()
.map(|(idx, err)| format!("[{idx}] {}", crate::error::wire_message(&**err)))
.collect::<Vec<_>>()
.join("; ");
let serialized_losers: Vec<SerializedLoser> = losers
.iter()
.map(|err| {
match err
.downcast_ref::<OperationError>()
.and_then(|op| op.wire())
{
Some(wire) => SerializedLoser::from_wire(wire),
None => SerializedLoser::from_wire(&crate::error::wire_error_for(
&**err,
COMBINATOR_ERROR_TYPE,
)),
}
})
.collect();
#[expect(clippy::expect_used)]
let losers_json = serde_json::to_string(&serialized_losers)
.expect("a SerializedLoser always serializes to JSON");
crate::error::WireError::new(
Some(ALL_FAILED_ERROR_TYPE),
Some(format!(
"all {} futures failed: {loser_summary}",
losers.len()
)),
)
.with_error_data(Some(losers_json))
}
fn combinator_resumed(ctx: &DurableContext, wire_id: &str, name: Option<&str>) {
if let Some(plugins) = ctx.plugins() {
let info = combinator_info(ctx, wire_id, name).with_replay(true);
crate::plugin::operation_started(plugins, &info);
}
}
fn serialize_result<T: Serialize + ?Sized>(value: &T) -> Result<String, OperationError> {
serde_json::to_string(value)
.map_err(|e| combinator_internal_error(&format!("serialization failed: {e}")))
}
async fn checkpoint_collection_succeed(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
start: Option<SubmittedWrite>,
serialized: &str,
) -> Result<(), OperationError> {
if serialized.len() > CHECKPOINT_SIZE_LIMIT_BYTES {
checkpoint_succeed(ctx, wire_id, name, start, "", true).await
} else {
let preserve = ctx.preserves_children(1);
checkpoint_succeed(ctx, wire_id, name, start, serialized, preserve).await
}
}
async fn checkpoint_winner_succeed(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
start: Option<SubmittedWrite>,
winner: usize,
serialized: &str,
) -> Result<(), OperationError> {
if serialized.len() > CHECKPOINT_SIZE_LIMIT_BYTES {
#[expect(clippy::expect_used)]
let record =
serde_json::to_string(&RecordedWinner { winner }).expect("a RecordedWinner serializes");
checkpoint_succeed(ctx, wire_id, name, start, &record, true).await
} else {
checkpoint_succeed(ctx, wire_id, name, start, serialized, false).await
}
}
async fn replay_recorded_winner<O: Serialize + DeserializeOwned + Send + 'static>(
ctx: &DurableContext,
positional_id: &str,
futures: Vec<DurableFuture<O>>,
) -> Result<O, OperationError> {
if let Some(message) = unclaimed_input_message(positional_id, &futures) {
return Err(reject_unclaimed(ctx, &message));
}
let RecordedWinner { winner } = ctx
.checkpoint_result_payload(positional_id)
.and_then(|payload| serde_json::from_str(&payload).ok())
.ok_or_else(|| combinator_internal_error("replay: malformed winner record"))?;
let count = futures.len();
let Some(input) = futures.into_iter().nth(winner) else {
return Err(combinator_internal_error(&format!(
"replay: recorded winner index {winner} is out of range for {count} inputs"
)));
};
let value = input.await?;
let serialized = serialize_result(&value)?;
drop(value);
replay_single_success::<O>(Some(&serialized))
}
async fn checkpoint_succeed(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
start: Option<SubmittedWrite>,
payload: &str,
replay_children: bool,
) -> Result<(), OperationError> {
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
ctx.await_retained_start(wire_id, start, OperationAction::Succeed)
.await?;
fatal_gate(ctx)?;
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Context)
.sub_type(COMBINATOR_SUB_TYPE.to_owned())
.action(OperationAction::Succeed)
.payload(payload.to_owned());
if replay_children {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
if let Some(n) = name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = ctx.plugins() {
let info = combinator_info(ctx, wire_id, name)
.with_status(crate::OperationStatus::Succeeded)
.with_result(Some(payload.to_owned()))
.with_replay(false);
crate::plugin::operation_ended(plugins, &info);
}
Ok(())
}
async fn checkpoint_fail(
ctx: &DurableContext,
wire_id: &str,
name: Option<&str>,
start: Option<SubmittedWrite>,
error: &crate::error::WireError,
) -> Result<(), OperationError> {
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
ctx.await_retained_start(wire_id, start, OperationAction::Fail)
.await?;
fatal_gate(ctx)?;
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Context)
.sub_type(COMBINATOR_SUB_TYPE.to_owned())
.action(OperationAction::Fail)
.error(error.to_error_object());
if let Some(n) = name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
if ctx.preserves_children(1) {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = ctx.plugins() {
let info = combinator_info(ctx, wire_id, name)
.with_status(crate::OperationStatus::Failed)
.with_error(Some(error.clone()))
.with_replay(false);
crate::plugin::operation_ended(plugins, &info);
}
Ok(())
}
fn combinator_internal_error(message: &str) -> OperationError {
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::Internal,
vec![message.to_owned().into()],
)))
}
fn combinator_empty_input_error() -> OperationError {
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::EmptyInput,
Vec::new(),
)))
}
fn replay_vec_success<O: DeserializeOwned>(
result: Option<&String>,
) -> Result<Vec<O>, OperationError> {
let payload = result
.ok_or_else(|| combinator_internal_error("replay: succeeded record has no payload"))?;
serde_json::from_str(payload)
.map_err(|e| combinator_internal_error(&format!("replay deserialization failed: {e}")))
}
fn replay_single_success<O: DeserializeOwned>(
result: Option<&String>,
) -> Result<O, OperationError> {
let payload = result
.ok_or_else(|| combinator_internal_error("replay: succeeded record has no payload"))?;
serde_json::from_str(payload)
.map_err(|e| combinator_internal_error(&format!("replay deserialization failed: {e}")))
}
fn replay_outcomes_success<O: DeserializeOwned>(
result: Option<&String>,
) -> Result<Vec<Result<O, OperationError>>, OperationError> {
let payload = result
.ok_or_else(|| combinator_internal_error("replay: succeeded record has no payload"))?;
let serialized: Vec<SerializedOutcome<O>> = serde_json::from_str(payload)
.map_err(|e| combinator_internal_error(&format!("replay deserialization failed: {e}")))?;
let outcomes = serialized
.into_iter()
.map(|s| match s {
SerializedOutcome::Fulfilled { value } => Ok(value),
SerializedOutcome::Rejected {
message,
error_type,
error_data,
stack_trace,
} => {
let wire = crate::error::WireError::new(error_type, Some(message))
.with_error_data(error_data)
.with_stack_trace(stack_trace);
Err(OperationError::from_kind(OperationErrorKind::ChildContext(
ChildContextError::new(
ChildContextErrorKind::ChildFailed,
Some(crate::error::ReplayedFailure::source_from(wire.clone())),
),
))
.with_wire(wire))
}
})
.collect();
Ok(outcomes)
}
fn replay_combinator_failure(wire: crate::error::WireError, wire_id: &str) -> OperationError {
if wire.error_type() == Some(EMPTY_INPUT_ERROR_TYPE) {
return combinator_empty_input_error()
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire);
}
let recorded_join_index = wire
.error_type()
.and_then(|t| t.strip_prefix(JOIN_FAILED_ERROR_TYPE_PREFIX))
.and_then(|suffix| suffix.parse::<usize>().ok());
let (kind, failures) = if let Some(failed_index) = recorded_join_index {
(
CombinatorErrorKind::JoinFailed(crate::error::JoinFailed::new(failed_index)),
vec![crate::error::ReplayedFailure::source_from(wire.clone())],
)
} else if wire.error_type() == Some(ALL_FAILED_ERROR_TYPE)
&& let Some(losers) = wire
.error_data()
.and_then(|data| serde_json::from_str::<Vec<SerializedLoser>>(data).ok())
.filter(|losers| !losers.is_empty())
{
let failures = losers
.into_iter()
.map(|loser| crate::error::ReplayedFailure::source_from(loser.into_wire()))
.collect();
(CombinatorErrorKind::AllFailed, failures)
} else if wire.error_type() == Some(FIRST_SETTLED_FAILED_ERROR_TYPE) {
(
CombinatorErrorKind::FirstSettledFailed,
vec![crate::error::ReplayedFailure::source_from(wire.clone())],
)
} else {
(
CombinatorErrorKind::Internal,
vec![crate::error::ReplayedFailure::source_from(wire.clone())],
)
};
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
kind, failures,
)))
.with_operation(wire_id, CheckpointStatus::Failed.wire_str())
.with_wire(wire)
}
#[cfg(test)]
#[expect(
clippy::panic,
clippy::indexing_slicing,
clippy::items_after_statements
)]
#[expect(clippy::disallowed_methods)]
mod tests {
use super::*;
use crate::client::InMemoryExecutionClient;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
use std::future::Future;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
fn test_ctx(log: CheckpointLog) -> DurableContext {
DurableContext::new_root(
"arn:aws:lambda:us-east-1:123456789012:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
)
}
fn test_ctx_with_client(log: CheckpointLog) -> DurableContext {
use crate::client::{
CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput,
};
use std::future::Future;
use std::pin::Pin;
#[derive(Debug)]
struct MockClient;
impl ExecutionClient for MockClient {
fn checkpoint(
&self,
_arn: &str,
_token: &str,
_updates: Vec<aws_sdk_lambda::types::OperationUpdate>,
) -> Pin<
Box<
dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_,
>,
> {
Box::pin(async {
Ok(CheckpointOutput {
checkpoint_token: Some("token-2".to_owned()),
updated_operations: vec![],
next_marker: None,
})
})
}
fn get_state(
&self,
_arn: &str,
_token: &str,
_marker: Option<&str>,
) -> Pin<
Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
> {
Box::pin(async { Ok(GetStateOutput::new(vec![], None)) })
}
}
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:123456789012:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::new(MockClient),
"token-1".to_owned(),
)
}
fn succeeded_record(positional_id: &str, result: &str) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some(result.to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
fn failed_record(
positional_id: &str,
err_type: &str,
err_msg: &str,
) -> (String, CheckpointRecord) {
failed_record_with_data(positional_id, err_type, err_msg, None)
}
fn failed_record_with_data(
positional_id: &str,
err_type: &str,
err_msg: &str,
error_data: Option<String>,
) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Failed,
result: None,
error_type: Some(err_type.to_owned()),
error_message: Some(err_msg.to_owned()),
error_data,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
#[tokio::test]
async fn try_join_all_success_live() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a = DurableFuture::from_async(async { Ok(1) });
let b = DurableFuture::from_async(async { Ok(2) });
let c = DurableFuture::from_async(async { Ok(3) });
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution {
ctx,
op_id,
name: Some("gather".to_owned()),
futures: vec![a, b, c],
};
let result = exec.execute().await.unwrap();
assert_eq!(result, vec![1, 2, 3]);
}
#[tokio::test]
async fn try_join_all_fail_fast() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a = DurableFuture::from_async(async { Ok(1) });
let b: DurableFuture<i32> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["boom".to_owned().into()],
),
)))
});
let c = DurableFuture::from_async(async {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok(3)
});
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![a, b, c],
};
let err = exec.execute().await.unwrap_err();
assert!(matches!(err.kind(), OperationErrorKind::Combinator(_)));
}
#[tokio::test]
async fn try_join_all_replay_success() {
let (wire_id, record) = succeeded_record("1", "[10,20,30]");
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id(); let exec = TryJoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![], };
let result = exec.execute().await.unwrap();
assert_eq!(result, vec![10, 20, 30]);
}
#[tokio::test]
async fn try_join_all_replay_failure() {
let (wire_id, record) =
failed_record("1", "CombinatorError", "join failed at index 1: boom");
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
assert!(matches!(err.kind(), OperationErrorKind::Combinator(_)));
}
fn recording_ctx(log: CheckpointLog) -> (DurableContext, Arc<InMemoryExecutionClient>) {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
);
(ctx, client)
}
async fn panic_of<F, Fut>(run: F) -> (Option<&'static str>, bool)
where
F: FnOnce(DurableContext) -> Fut,
Fut: Future<Output = ()> + Send + 'static,
{
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let handler = run(ctx);
let outcome = crate::driver::drive_outcome(
async move {
handler.await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
ownership,
signal,
root,
None,
)
.await;
let Err(payload) = outcome else {
panic!("the input panic must reach the invocation driver, got {outcome:?}");
};
let wrote_fail = client
.recorded_updates()
.iter()
.any(|u| u.action() == &aws_sdk_lambda::types::OperationAction::Fail);
(
crate::task::test_support::str_payload(payload.as_ref()),
wrote_fail,
)
}
#[tokio::test]
async fn try_join_all_input_panic_propagates_payload() {
let (message, wrote_fail) = panic_of(|ctx| async move {
let a = DurableFuture::from_async(async {
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
Ok(1)
});
let b: DurableFuture<i32> = DurableFuture::from_async(async { panic!("boom-idx1") });
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let _ = exec.execute().await;
})
.await;
assert_eq!(message, Some("boom-idx1"));
assert!(!wrote_fail, "a panic records no FAIL");
}
#[tokio::test]
async fn join_all_input_panic_propagates_payload() {
let (message, wrote_fail) = panic_of(|ctx| async move {
let a = DurableFuture::from_async(async {
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
Ok(1)
});
let b: DurableFuture<i32> = DurableFuture::from_async(async { panic!("boom-idx1") });
let op_id = ctx.mint_id();
let exec = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let _ = exec.execute().await;
})
.await;
assert_eq!(message, Some("boom-idx1"));
assert!(!wrote_fail, "a panic records no FAIL");
}
#[tokio::test]
async fn select_ok_input_panic_propagates_payload() {
let (message, wrote_fail) = panic_of(|ctx| async move {
let a: DurableFuture<String> = DurableFuture::from_async(async { panic!("boom-idx0") });
let b: DurableFuture<String> = DurableFuture::from_async(std::future::pending());
let op_id = ctx.mint_id();
let exec = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let _ = exec.execute().await;
})
.await;
assert_eq!(message, Some("boom-idx0"));
assert!(!wrote_fail, "a panic records no FAIL");
}
#[tokio::test]
async fn race_input_panic_propagates_payload() {
let (message, wrote_fail) = panic_of(|ctx| async move {
let a: DurableFuture<String> = DurableFuture::from_async(async { panic!("boom-idx0") });
let b: DurableFuture<String> = DurableFuture::from_async(std::future::pending());
let op_id = ctx.mint_id();
let exec = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let _ = exec.execute().await;
})
.await;
assert_eq!(message, Some("boom-idx0"));
assert!(!wrote_fail, "a panic records no FAIL");
}
#[tokio::test]
async fn try_join_all_join_failed_record_replays() {
let (wire_id, record) = failed_record("1", "CombinatorError.JoinFailed.1", "input failed");
let ctx = test_ctx(CheckpointLog::from_records(vec![(wire_id, record)]));
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
let OperationErrorKind::Combinator(ce) = err.kind() else {
panic!("expected Combinator, got: {:?}", err.kind());
};
let CombinatorErrorKind::JoinFailed(details) = ce.kind() else {
panic!("expected JoinFailed, got: {:?}", ce.kind());
};
assert_eq!(details.failed_index(), 1);
}
#[tokio::test]
async fn join_all_live_payload_keeps_wire_format() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let ok = DurableFuture::from_async(async { Ok(1_u32) });
let bad: DurableFuture<u32> = DurableFuture::from_async(async {
Err(
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["boom".to_owned().into()],
)))
.with_wire(crate::error::WireError::new(
Some("Error".to_owned()),
Some("boom".to_owned()),
)),
)
});
let op_id = ctx.mint_id();
JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ok, bad],
}
.execute()
.await
.expect("the combinator itself succeeds");
assert_eq!(
succeed_payload(&client),
r#"[{"status":"fulfilled","value":1},{"status":"rejected","message":"boom","error_type":"Error"}]"#
);
}
fn assert_no_runtime_error<F, Fut, T>(run: F)
where
F: FnOnce(DurableContext) -> Fut + Send + 'static,
Fut: Future<Output = Result<T, OperationError>>,
T: Send + std::fmt::Debug + 'static,
{
let polled = crate::task::test_support::poll_once_without_runtime(move || {
run(test_ctx_with_client(CheckpointLog::empty()))
});
let std::task::Poll::Ready(Err(err)) = polled else {
panic!("expected an immediate error, got {polled:?}");
};
assert!(
format!("{err:#}").contains(crate::task::NO_RUNTIME_MESSAGE),
"unexpected error: {err:#}"
);
}
fn two_inputs() -> Vec<DurableFuture<i32>> {
vec![
DurableFuture::from_async(async { Ok(1) }),
DurableFuture::from_async(async { Ok(2) }),
]
}
#[test]
fn first_poll_input_without_runtime_returns_error() {
assert_no_runtime_error(|ctx| {
let op_id = ctx.mint_id();
let input = DurableFuture::for_first_poll_operation(&ctx, async { Ok(1) });
TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![input],
}
.execute()
});
}
#[test]
fn try_join_all_without_runtime_returns_error() {
assert_no_runtime_error(|ctx| {
let op_id = ctx.mint_id();
TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: two_inputs(),
}
.execute()
});
}
#[test]
fn join_all_without_runtime_returns_error() {
assert_no_runtime_error(|ctx| {
let op_id = ctx.mint_id();
JoinAllExecution {
ctx,
op_id,
name: None,
futures: two_inputs(),
}
.execute()
});
}
#[test]
fn select_ok_without_runtime_returns_error() {
assert_no_runtime_error(|ctx| {
let op_id = ctx.mint_id();
SelectOkExecution {
ctx,
op_id,
name: None,
futures: two_inputs(),
}
.execute()
});
}
#[test]
fn race_without_runtime_returns_error() {
assert_no_runtime_error(|ctx| {
let op_id = ctx.mint_id();
RaceExecution {
ctx,
op_id,
name: None,
futures: two_inputs(),
}
.execute()
});
}
#[tokio::test]
async fn join_all_mixed_outcomes() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a = DurableFuture::from_async(async { Ok(1) });
let b: DurableFuture<i32> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["task-b-failed".to_owned().into()],
),
)))
});
let c = DurableFuture::from_async(async { Ok(3) });
let op_id = ctx.mint_id();
let exec = JoinAllExecution {
ctx,
op_id,
name: Some("collect".to_owned()),
futures: vec![a, b, c],
};
let result = exec.execute().await.unwrap();
assert_eq!(result.len(), 3);
assert!(matches!(&result[0], Ok(1)));
assert!(result[1].is_err());
assert!(matches!(&result[2], Ok(3)));
}
#[tokio::test]
async fn join_all_replay_with_error_aware_serdes() {
let payload = r#"[{"status":"fulfilled","value":42},{"status":"rejected","message":"oops"},{"status":"fulfilled","value":99}]"#;
let (wire_id, record) = succeeded_record("1", payload);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = JoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![],
};
let result = exec.execute().await.unwrap();
assert_eq!(result.len(), 3);
assert!(matches!(&result[0], Ok(42)));
assert!(matches!(&result[1], Err(e) if format!("{e:#}").contains("oops")));
assert!(matches!(&result[2], Ok(99)));
}
#[tokio::test]
async fn select_ok_first_success() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a = DurableFuture::from_async(async { Ok("winner".to_owned()) });
let b: DurableFuture<String> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["fail".to_owned().into()],
),
)))
});
let op_id = ctx.mint_id();
let exec = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let result = exec.execute().await.unwrap();
assert_eq!(result, "winner");
}
#[tokio::test]
async fn select_ok_all_failed_aggregate() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a: DurableFuture<String> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["err-a".to_owned().into()],
),
)))
});
let b: DurableFuture<String> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["err-b".to_owned().into()],
),
)))
});
let op_id = ctx.mint_id();
let exec = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => match ce.kind() {
CombinatorErrorKind::AllFailed => {
assert_eq!(ce.failures().len(), 2);
}
other => panic!("expected AllFailed, got: {other:?}"),
},
other => panic!("expected Combinator, got: {other:?}"),
}
let wire = err.wire().expect("live AllFailed carries wire record");
let payload = wire.error_data().expect("AllFailed persists losers");
let losers: Vec<SerializedLoser> =
serde_json::from_str(payload).expect("losers payload decodes");
assert_eq!(losers.len(), 2);
assert!(
losers[0]
.message
.as_deref()
.is_some_and(|m| m.contains("err-a")),
"loser 0 keeps its own message: {:?}",
losers[0].message
);
assert!(
losers[1]
.message
.as_deref()
.is_some_and(|m| m.contains("err-b")),
"loser 1 keeps its own message: {:?}",
losers[1].message
);
}
#[tokio::test]
async fn select_ok_all_failed_replay_rebuilds_each_recorded_loser() {
let losers_payload = serde_json::to_string(&vec![
SerializedLoser {
error_type: Some("TypeA".to_owned()),
message: Some("boom-a".to_owned()),
error_data: Some(r#"{"a":1}"#.to_owned()),
stack_trace: vec!["frame-a0".to_owned(), "frame-a1".to_owned()],
},
SerializedLoser {
error_type: Some("TypeB".to_owned()),
message: Some("boom-b".to_owned()),
error_data: None,
stack_trace: Vec::new(),
},
])
.expect("payload serializes");
let (wire_id, record) = failed_record_with_data(
"1",
"CombinatorError.AllFailed",
"all 2 futures failed: [0] boom-a; [1] boom-b",
Some(losers_payload),
);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = SelectOkExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![], };
let err = exec.execute().await.unwrap_err();
let OperationErrorKind::Combinator(ce) = err.kind() else {
panic!("expected Combinator, got: {:?}", err.kind());
};
assert!(
matches!(ce.kind(), CombinatorErrorKind::AllFailed),
"expected AllFailed, got: {:?}",
ce.kind()
);
let failures = ce.failures();
assert_eq!(failures.len(), 2);
let replayed = |idx: usize| {
failures[idx]
.downcast_ref::<crate::error::ReplayedFailure>()
.expect("replayed loser is a ReplayedFailure")
.wire()
.clone()
};
let first = replayed(0);
assert_eq!(first.error_type(), Some("TypeA"));
assert_eq!(first.error_message(), Some("boom-a"));
assert_eq!(first.error_data(), Some(r#"{"a":1}"#));
assert_eq!(
first.stack_trace(),
["frame-a0".to_owned(), "frame-a1".to_owned()]
);
let second = replayed(1);
assert_eq!(second.error_type(), Some("TypeB"));
assert_eq!(second.error_message(), Some("boom-b"));
assert_eq!(second.error_data(), None);
assert_eq!(second.stack_trace(), &[] as &[String]);
}
#[tokio::test]
async fn select_ok_all_failed_replay_without_valid_losers_is_internal() {
let message = "all 3 futures failed: [0] a; [1] b; [2] c";
for error_data in [None, Some("not json".to_owned()), Some("[]".to_owned())] {
let (wire_id, record) = failed_record_with_data(
"1",
"CombinatorError.AllFailed",
message,
error_data.clone(),
);
let ctx = test_ctx(CheckpointLog::from_records(vec![(wire_id, record)]));
let op_id = ctx.mint_id();
let a: DurableFuture<String> =
DurableFuture::from_async(async { panic!("never runs") });
let b: DurableFuture<String> =
DurableFuture::from_async(async { panic!("never runs") });
let c: DurableFuture<String> =
DurableFuture::from_async(async { panic!("never runs") });
let exec = SelectOkExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![a, b, c],
};
let err = exec.execute().await.unwrap_err();
let OperationErrorKind::Combinator(ce) = err.kind() else {
panic!("expected Combinator, got: {:?}", err.kind());
};
assert!(
matches!(ce.kind(), CombinatorErrorKind::Internal),
"{error_data:?}: expected Internal, got: {:?}",
ce.kind()
);
assert_eq!(ce.failures().len(), 1, "{error_data:?}");
let failure = ce.failures()[0]
.downcast_ref::<crate::error::ReplayedFailure>()
.expect("replayed failure is a ReplayedFailure")
.wire();
assert_eq!(failure.error_message(), Some(message), "{error_data:?}");
}
}
#[tokio::test]
async fn select_ok_replay_returns_winner() {
let payload = r#""the-winner""#;
let (wire_id, record) = succeeded_record("1", payload);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = SelectOkExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![], };
let result = exec.execute().await.unwrap();
assert_eq!(result, "the-winner");
}
#[tokio::test]
async fn race_first_settled_success() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a = DurableFuture::from_async(async { Ok("fast".to_owned()) });
let b = DurableFuture::from_async(async {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok("slow".to_owned())
});
let op_id = ctx.mint_id();
let exec = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let result = exec.execute().await.unwrap();
assert_eq!(result, "fast");
}
#[tokio::test]
async fn race_failure_wins() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let a: DurableFuture<String> = DurableFuture::from_async(async {
Err(OperationError::from_kind(OperationErrorKind::Combinator(
CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["fast-fail".to_owned().into()],
),
)))
});
let b = DurableFuture::from_async(async {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok("slow-success".to_owned())
});
let op_id = ctx.mint_id();
let exec = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => match ce.kind() {
CombinatorErrorKind::FirstSettledFailed => {
let message = crate::error::chain_string(ce);
assert!(
message.contains("fast-fail"),
"loser's error must be preserved: {message}"
);
}
other => panic!("expected FirstSettledFailed, got: {other:?}"),
},
other => panic!("expected Combinator, got: {other:?}"),
}
let wire = err.wire().expect("first-settled failure carries wire");
assert_eq!(
wire.error_type(),
Some("CombinatorError.FirstSettledFailed")
);
assert!(
wire.stack_trace().is_empty(),
"the SDK captures no stack trace for a first-settled failure"
);
}
#[tokio::test]
async fn race_replay_failure_reproduces_first_settled_failed() {
let (wire_id, record) = failed_record(
"1",
"CombinatorError.FirstSettledFailed",
"step failed: fast-fail",
);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = RaceExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => match ce.kind() {
CombinatorErrorKind::FirstSettledFailed => {
let message = crate::error::chain_string(ce);
assert!(
message.contains("fast-fail"),
"replayed message must match the recorded one: {message}"
);
}
other => panic!("expected FirstSettledFailed, got: {other:?}"),
},
other => panic!("expected Combinator, got: {other:?}"),
}
}
#[tokio::test]
async fn race_empty_input() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let op_id = ctx.mint_id();
let exec = RaceExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => {
assert!(
matches!(ce.kind(), CombinatorErrorKind::EmptyInput),
"expected EmptyInput, got: {:?}",
ce.kind()
);
}
other => panic!("expected Combinator, got: {other:?}"),
}
assert!(err.operation_id().is_some(), "live EmptyInput has op id");
assert_eq!(err.status(), Some("FAILED"));
let wire = err.wire().expect("live EmptyInput carries wire record");
assert_eq!(wire.error_type(), Some("CombinatorError.EmptyInput"));
assert!(
wire.stack_trace().is_empty(),
"the SDK captures no stack trace for an empty input"
);
}
#[tokio::test]
async fn select_ok_empty_input() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let op_id = ctx.mint_id();
let exec = SelectOkExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => {
assert!(
matches!(ce.kind(), CombinatorErrorKind::EmptyInput),
"expected EmptyInput, got: {:?}",
ce.kind()
);
}
other => panic!("expected Combinator, got: {other:?}"),
}
assert!(err.operation_id().is_some(), "live EmptyInput has op id");
assert_eq!(err.status(), Some("FAILED"));
let wire = err.wire().expect("live EmptyInput carries wire record");
assert_eq!(wire.error_type(), Some("CombinatorError.EmptyInput"));
assert_eq!(wire.stack_trace(), [] as [String; 0]);
}
#[tokio::test]
async fn empty_input_replay_reproduces_empty_input() {
let (wire_id, record) = failed_record(
"1",
"CombinatorError.EmptyInput",
"race called with no futures",
);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = RaceExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![],
};
let err = exec.execute().await.unwrap_err();
match err.kind() {
OperationErrorKind::Combinator(ce) => {
assert!(
matches!(ce.kind(), CombinatorErrorKind::EmptyInput),
"expected EmptyInput, got: {:?}",
ce.kind()
);
}
other => panic!("expected Combinator, got: {other:?}"),
}
}
#[tokio::test]
async fn try_join_all_empty_returns_empty_vec() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![],
};
let result = exec.execute().await.unwrap();
assert!(result.is_empty(), "try_join_all([]) must be Ok(empty)");
}
#[tokio::test]
async fn join_all_empty_returns_empty_vec() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let op_id = ctx.mint_id();
let exec = JoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![],
};
let result = exec.execute().await.unwrap();
assert!(result.is_empty(), "join_all([]) must be Ok(empty)");
}
#[tokio::test]
async fn race_replay_returns_recorded_winner() {
let payload = r#""replay-winner""#;
let (wire_id, record) = succeeded_record("1", payload);
let log = CheckpointLog::from_records(vec![(wire_id, record)]);
let ctx = test_ctx(log);
let op_id = ctx.mint_id();
let exec = RaceExecution::<String> {
ctx,
op_id,
name: None,
futures: vec![], };
let result = exec.execute().await.unwrap();
assert_eq!(result, "replay-winner");
}
#[tokio::test]
async fn race_loser_drop_cancels() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let cancelled = Arc::new(AtomicBool::new(false));
let cancelled_clone = Arc::clone(&cancelled);
struct DropGuard(Arc<AtomicBool>);
impl Drop for DropGuard {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
let a = DurableFuture::from_async(async { Ok("winner".to_owned()) });
let b = DurableFuture::from_async(async move {
let _guard = DropGuard(cancelled_clone);
std::future::pending::<()>().await;
Ok("never".to_owned())
});
let op_id = ctx.mint_id();
let exec = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let result = exec.execute().await.unwrap();
assert_eq!(result, "winner");
tokio::task::yield_now().await;
assert!(
cancelled.load(Ordering::SeqCst),
"loser future should have been dropped (cancelled)"
);
}
#[tokio::test]
async fn select_ok_loser_drop_cancels() {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let cancelled = Arc::new(AtomicBool::new(false));
let cancelled_clone = Arc::clone(&cancelled);
struct DropGuard(Arc<AtomicBool>);
impl Drop for DropGuard {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
let a = DurableFuture::from_async(async { Ok("winner".to_owned()) });
let b = DurableFuture::from_async(async move {
let _guard = DropGuard(cancelled_clone);
std::future::pending::<()>().await;
Ok("never".to_owned())
});
let op_id = ctx.mint_id();
let exec = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
let result = exec.execute().await.unwrap();
assert_eq!(result, "winner");
tokio::task::yield_now().await;
assert!(
cancelled.load(Ordering::SeqCst),
"loser future should have been dropped (cancelled)"
);
}
#[tokio::test]
async fn combinator_ids_deterministic_under_reversed_order() {
let ctx1 = test_ctx_with_client(CheckpointLog::empty());
let ctx2 = test_ctx_with_client(CheckpointLog::empty());
let id1 = ctx1.mint_id();
let id2 = ctx2.mint_id();
assert_eq!(id1.positional(), id2.positional());
assert_eq!(id1.wire(), id2.wire());
let id1b = ctx1.mint_id();
let id2b = ctx2.mint_id();
assert_eq!(id1b.positional(), id2b.positional());
assert_eq!(id1b.wire(), id2b.wire());
}
#[tokio::test]
async fn tokio_join_interop_with_combinator_builders() {
let _ctx = test_ctx_with_client(CheckpointLog::empty());
let f1 = DurableFuture::from_async(async { Ok(1_i32) });
let f2 = DurableFuture::from_async(async { Ok(2_i32) });
let (r1, r2) = tokio::join!(f1, f2);
assert_eq!(r1.unwrap(), 1);
assert_eq!(r2.unwrap(), 2);
}
#[tokio::test]
async fn try_join_all_ownership_blessed() {
let result = tokio::spawn(async {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let ctx_a = ctx.clone();
let a = DurableFuture::from_async(async move {
ctx_a.enforce_task_ownership()?;
Ok(1)
});
let ctx_b = ctx.clone();
let b = DurableFuture::from_async(async move {
ctx_b.enforce_task_ownership()?;
Ok(2)
});
let op_id = ctx.mint_id();
let exec = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
exec.execute().await
})
.await
.unwrap();
let values = result.unwrap();
assert_eq!(values, vec![1, 2]);
}
#[tokio::test]
async fn join_all_ownership_blessed_no_rejections() {
let result = tokio::spawn(async {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let ctx_a = ctx.clone();
let a = DurableFuture::from_async(async move {
ctx_a.enforce_task_ownership()?;
Ok(10)
});
let ctx_b = ctx.clone();
let b = DurableFuture::from_async(async move {
ctx_b.enforce_task_ownership()?;
Ok(20)
});
let ctx_c = ctx.clone();
let c = DurableFuture::from_async(async move {
ctx_c.enforce_task_ownership()?;
Ok(30)
});
let op_id = ctx.mint_id();
let exec = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![a, b, c],
};
exec.execute().await
})
.await
.unwrap();
let settled = result.unwrap();
assert_eq!(settled.len(), 3);
for (i, s) in settled.iter().enumerate() {
assert!(
s.is_ok(),
"branch {i} must be Ok, not Err with an ownership error"
);
}
assert!(matches!(&settled[0], Ok(10)));
assert!(matches!(&settled[1], Ok(20)));
assert!(matches!(&settled[2], Ok(30)));
}
#[tokio::test]
async fn select_ok_ownership_blessed() {
let result = tokio::spawn(async {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let ctx_a = ctx.clone();
let a = DurableFuture::from_async(async move {
ctx_a.enforce_task_ownership()?;
Ok("winner".to_owned())
});
let ctx_b = ctx.clone();
let b = DurableFuture::from_async(async move {
ctx_b.enforce_task_ownership()?;
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
Ok("slower".to_owned())
});
let op_id = ctx.mint_id();
let exec = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
exec.execute().await
})
.await
.unwrap();
let value = result.unwrap();
assert_eq!(value, "winner");
}
#[tokio::test]
async fn race_ownership_blessed() {
let result = tokio::spawn(async {
let ctx = test_ctx_with_client(CheckpointLog::empty());
let ctx_a = ctx.clone();
let a = DurableFuture::from_async(async move {
ctx_a.enforce_task_ownership()?;
Ok("fast".to_owned())
});
let ctx_b = ctx.clone();
let b = DurableFuture::from_async(async move {
ctx_b.enforce_task_ownership()?;
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok("slow".to_owned())
});
let op_id = ctx.mint_id();
let exec = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![a, b],
};
exec.execute().await
})
.await
.unwrap();
let value = result.unwrap();
assert_eq!(value, "fast");
}
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Tagged {
value: u32,
#[serde(skip)]
cached: Option<String>,
}
fn tagged_input() -> DurableFuture<Tagged> {
DurableFuture::from_async(async {
Ok(Tagged {
value: 7,
cached: Some("live-only".to_owned()),
})
})
}
fn failing_input<O: Send + 'static>() -> DurableFuture<O> {
DurableFuture::from_async(async {
Err(
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::Internal,
vec!["input failed".to_owned().into()],
)))
.with_wire(crate::error::WireError::new(
Some("InputFailure"),
Some("input failed"),
)),
)
})
}
fn succeed_payload(client: &InMemoryExecutionClient) -> String {
let succeeds: Vec<String> = client
.recorded_updates()
.iter()
.filter(|u| u.action() == &aws_sdk_lambda::types::OperationAction::Succeed)
.map(|u| u.payload().unwrap_or_default().to_owned())
.collect();
assert_eq!(succeeds.len(), 1, "one SUCCEED update: {succeeds:?}");
succeeds[0].clone()
}
fn replay_ctx(payload: &str) -> DurableContext {
let (wire_id, record) = succeeded_record("1", payload);
test_ctx(CheckpointLog::from_records(vec![(wire_id, record)]))
}
#[tokio::test]
async fn try_join_all_live_value_equals_replay() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let live = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![tagged_input(), tagged_input()],
}
.execute()
.await
.unwrap();
let payload = succeed_payload(&client);
let ctx = replay_ctx(&payload);
let op_id = ctx.mint_id();
let replayed = TryJoinAllExecution::<Tagged> {
ctx,
op_id,
name: None,
futures: vec![],
}
.execute()
.await
.unwrap();
assert_eq!(live, replayed);
assert!(live.iter().all(|t| t.cached.is_none() && t.value == 7));
}
fn render_settled(settled: &[Result<Tagged, OperationError>]) -> Vec<String> {
settled
.iter()
.map(|s| match s {
Ok(t) => format!("ok:{}:{:?}", t.value, t.cached),
Err(err) => format!(
"err:{:?}:{:?}:{:?}",
err.kind(),
err.wire().and_then(crate::error::WireError::error_type),
err.wire().and_then(crate::error::WireError::error_message),
),
})
.collect()
}
async fn join_all_live() -> (Vec<Result<Tagged, OperationError>>, String) {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let live = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![tagged_input(), failing_input()],
}
.execute()
.await
.unwrap();
(live, succeed_payload(&client))
}
#[tokio::test]
async fn join_all_live_value_equals_replay() {
let (live, payload) = join_all_live().await;
let ctx = replay_ctx(&payload);
let op_id = ctx.mint_id();
let replayed = JoinAllExecution::<Tagged> {
ctx,
op_id,
name: None,
futures: vec![],
}
.execute()
.await
.unwrap();
assert_eq!(render_settled(&live), render_settled(&replayed));
assert!(live.iter().all(|s| match s {
Ok(t) => t.cached.is_none(),
Err(_) => true,
}));
}
#[tokio::test]
async fn join_all_live_rejection_is_child_failed() {
let (live, payload) = join_all_live().await;
let Some(Err(err)) = live.get(1) else {
panic!("item 1 must be rejected: {live:?}");
};
let OperationErrorKind::ChildContext(child) = err.kind() else {
panic!("expected a ChildContext error, got {:?}", err.kind());
};
assert!(matches!(child.kind(), ChildContextErrorKind::ChildFailed));
let recorded: serde_json::Value = serde_json::from_str(&payload).unwrap();
let recorded_type = recorded[1]["error_type"].as_str();
assert_eq!(recorded_type, Some("InputFailure"));
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
recorded_type
);
}
#[tokio::test]
async fn select_ok_live_value_equals_replay() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let live = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![tagged_input()],
}
.execute()
.await
.unwrap();
let payload = succeed_payload(&client);
let ctx = replay_ctx(&payload);
let op_id = ctx.mint_id();
let replayed = SelectOkExecution::<Tagged> {
ctx,
op_id,
name: None,
futures: vec![],
}
.execute()
.await
.unwrap();
assert_eq!(live, replayed);
assert_eq!(live.cached, None);
}
#[tokio::test]
async fn race_live_value_equals_replay() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let live = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![tagged_input()],
}
.execute()
.await
.unwrap();
let payload = succeed_payload(&client);
let ctx = replay_ctx(&payload);
let op_id = ctx.mint_id();
let replayed = RaceExecution::<Tagged> {
ctx,
op_id,
name: None,
futures: vec![],
}
.execute()
.await
.unwrap();
assert_eq!(live, replayed);
assert_eq!(live.cached, None);
}
#[derive(Debug)]
struct Unreadable;
impl Serialize for Unreadable {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_u32(0)
}
}
impl<'de> serde::Deserialize<'de> for Unreadable {
fn deserialize<D: serde::Deserializer<'de>>(_deserializer: D) -> Result<Self, D::Error> {
Err(serde::de::Error::custom("unreadable"))
}
}
fn internal_message(err: &OperationError) -> String {
let OperationErrorKind::Combinator(ce) = err.kind() else {
panic!("expected a Combinator error, got {:?}", err.kind());
};
assert!(
matches!(ce.kind(), CombinatorErrorKind::Internal),
"expected Internal, got {:?}",
ce.kind()
);
ce.failures()[0].to_string()
}
#[tokio::test]
async fn try_join_all_live_decode_failure_matches_replay() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let live = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![DurableFuture::from_async(async { Ok(Unreadable) })],
}
.execute()
.await
.unwrap_err();
let payload = succeed_payload(&client);
assert_eq!(payload, "[0]");
let ctx = replay_ctx(&payload);
let op_id = ctx.mint_id();
let replayed = TryJoinAllExecution::<Unreadable> {
ctx,
op_id,
name: None,
futures: vec![],
}
.execute()
.await
.unwrap_err();
let live_message = internal_message(&live);
assert_eq!(live_message, internal_message(&replayed));
assert!(
live_message.starts_with("replay deserialization failed"),
"{live_message}"
);
}
fn succeed_update(client: &InMemoryExecutionClient) -> aws_sdk_lambda::types::OperationUpdate {
let succeeds: Vec<aws_sdk_lambda::types::OperationUpdate> = client
.recorded_updates()
.into_iter()
.filter(|u| u.action() == &aws_sdk_lambda::types::OperationAction::Succeed)
.collect();
assert_eq!(succeeds.len(), 1, "one SUCCEED update: {succeeds:?}");
succeeds[0].clone()
}
fn replay_children_of(update: &aws_sdk_lambda::types::OperationUpdate) -> Option<bool> {
update
.context_options()
.and_then(aws_sdk_lambda::types::ContextOptions::replay_children)
}
fn ready<O: Send + 'static>(value: O) -> DurableFuture<O> {
DurableFuture::from_async(async move { Ok(value) })
}
fn never<O: Send + 'static>() -> DurableFuture<O> {
DurableFuture::from_async(std::future::pending())
}
fn must_not_run<O: Send + 'static>() -> DurableFuture<O> {
DurableFuture::from_async(async { panic!("this input must not run") })
}
fn flagging(flag: &Arc<AtomicBool>, value: i32) -> DurableFuture<i32> {
let flag = Arc::clone(flag);
DurableFuture::from_async(async move {
flag.store(true, Ordering::SeqCst);
Ok(value)
})
}
fn depth_update(
client: &InMemoryExecutionClient,
action: &aws_sdk_lambda::types::OperationAction,
) -> aws_sdk_lambda::types::OperationUpdate {
let found: Vec<_> = client
.recorded_updates()
.into_iter()
.filter(|u| u.action() == action)
.collect();
assert_eq!(found.len(), 1, "one {action:?} update: {found:?}");
found[0].clone()
}
async fn depth_run<T>(
depth: u32,
run: impl FnOnce(DurableContext, OperationId) -> T,
) -> Arc<InMemoryExecutionClient>
where
T: std::future::Future<Output = ()>,
{
let (ctx, client) = DurableContext::test_context_with_depth(depth);
let op_id = ctx.mint_id();
run(ctx, op_id).await;
client
}
#[tokio::test]
async fn preserved_try_join_all_success_has_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let value = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(1), ready(2)],
}
.execute()
.await
.unwrap();
assert_eq!(value, vec![1, 2]);
})
.await;
let update = succeed_update(&client);
assert_eq!(replay_children_of(&update), Some(true));
assert_eq!(update.payload(), Some("[1,2]"));
}
#[tokio::test]
async fn preserved_join_all_success_has_flag() {
let run = |depth| {
depth_run(depth, |ctx, op_id| async move {
let outcomes = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(1), failing_input::<i32>()],
}
.execute()
.await
.unwrap();
assert_eq!(outcomes.len(), 2);
})
};
let plain = succeed_update(&*run(0).await);
let update = succeed_update(&*run(1).await);
assert_eq!(replay_children_of(&plain), None);
assert_eq!(replay_children_of(&update), Some(true));
assert_eq!(update.payload(), plain.payload());
}
#[tokio::test]
async fn preserved_race_success_has_no_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let value = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![never(), ready(20)],
}
.execute()
.await
.unwrap();
assert_eq!(value, 20);
})
.await;
let update = succeed_update(&client);
assert_eq!(update.payload(), Some("20"));
assert!(update.context_options().is_none());
}
#[tokio::test]
async fn preserved_select_ok_success_has_no_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let value = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![never(), ready(20)],
}
.execute()
.await
.unwrap();
assert_eq!(value, 20);
})
.await;
let update = succeed_update(&client);
assert_eq!(update.payload(), Some("20"));
assert!(update.context_options().is_none());
}
#[tokio::test]
async fn preserved_try_join_all_failure_has_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let result = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![failing_input::<i32>()],
}
.execute()
.await;
assert!(result.is_err());
})
.await;
let update = depth_update(&client, &aws_sdk_lambda::types::OperationAction::Fail);
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn preserved_race_failure_has_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let result = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![never(), failing_input::<i32>()],
}
.execute()
.await;
assert!(result.is_err());
})
.await;
let update = depth_update(&client, &aws_sdk_lambda::types::OperationAction::Fail);
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn preserved_select_ok_failure_has_flag() {
let client = depth_run(1, |ctx, op_id| async move {
let result = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![failing_input::<i32>(), failing_input::<i32>()],
}
.execute()
.await;
assert!(result.is_err());
})
.await;
let update = depth_update(&client, &aws_sdk_lambda::types::OperationAction::Fail);
assert_eq!(replay_children_of(&update), Some(true));
}
fn replay_children_ctx(
payload: Option<&str>,
) -> (DurableContext, Arc<InMemoryExecutionClient>) {
let (wire_id, mut record) = succeeded_record("1", "");
record.result = payload.map(str::to_owned);
record.replay_children = true;
recording_ctx(CheckpointLog::from_records(vec![(wire_id, record)]))
}
async fn try_join_all_one_string(len: usize) -> aws_sdk_lambda::types::OperationUpdate {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let value = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready("x".repeat(len))],
}
.execute()
.await
.unwrap();
assert_eq!(value, vec!["x".repeat(len)]);
succeed_update(&client)
}
#[tokio::test]
async fn succeed_write_inline_at_limit() {
let update = try_join_all_one_string(262_140).await;
let payload = update.payload().unwrap();
assert_eq!(payload.len(), 262_144);
assert_eq!(payload, format!("[\"{}\"]", "x".repeat(262_140)));
assert_eq!(replay_children_of(&update), None);
}
#[tokio::test]
async fn succeed_write_replay_children_above_limit() {
let update = try_join_all_one_string(262_141).await;
assert_eq!(update.payload(), Some(""));
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn join_all_oversized_writes_replay_children() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let big = "x".repeat(262_145);
let value = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(big.clone())],
}
.execute()
.await
.unwrap();
assert_eq!(value.len(), 1);
assert_eq!(value[0].as_ref().unwrap(), &big);
let update = succeed_update(&client);
assert_eq!(update.payload(), Some(""));
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn race_oversized_records_winner() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let big = "x".repeat(262_145);
let value = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![never(), ready(big.clone())],
}
.execute()
.await
.unwrap();
assert_eq!(value, big);
let update = succeed_update(&client);
assert_eq!(update.payload(), Some("{\"winner\":1}"));
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn select_ok_oversized_records_winner() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let big = "x".repeat(262_145);
let value = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![never(), ready(big.clone())],
}
.execute()
.await
.unwrap();
assert_eq!(value, big);
let update = succeed_update(&client);
assert_eq!(update.payload(), Some("{\"winner\":1}"));
assert_eq!(replay_children_of(&update), Some(true));
}
#[tokio::test]
async fn race_small_result_writes_inline() {
let (ctx, client) = recording_ctx(CheckpointLog::empty());
let op_id = ctx.mint_id();
let value = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![never(), ready("small".to_owned())],
}
.execute()
.await
.unwrap();
assert_eq!(value, "small");
let update = succeed_update(&client);
assert_eq!(update.payload(), Some("\"small\""));
assert_eq!(replay_children_of(&update), None);
}
#[tokio::test]
async fn try_join_all_replay_children_reconstructs_without_writing() {
let (ctx, client) = replay_children_ctx(Some(""));
let op_id = ctx.mint_id();
let value = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(1), ready(2)],
}
.execute()
.await
.unwrap();
assert_eq!(value, vec![1, 2]);
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[tokio::test]
async fn try_join_all_replay_children_input_failure_writes_nothing() {
let (ctx, client) = replay_children_ctx(Some(""));
let op_id = ctx.mint_id();
let err = TryJoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(1), failing_input()],
}
.execute()
.await
.unwrap_err();
let OperationErrorKind::Combinator(ce) = err.kind() else {
panic!("expected a Combinator error, got {:?}", err.kind());
};
let CombinatorErrorKind::JoinFailed(failed) = ce.kind() else {
panic!("expected JoinFailed, got {:?}", ce.kind());
};
assert_eq!(failed.failed_index(), 1);
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some("CombinatorError.JoinFailed.1")
);
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[tokio::test]
async fn join_all_replay_children_reconstructs_without_writing() {
let (ctx, client) = replay_children_ctx(Some(""));
let op_id = ctx.mint_id();
let value = JoinAllExecution {
ctx,
op_id,
name: None,
futures: vec![ready(1), failing_input()],
}
.execute()
.await
.unwrap();
assert_eq!(value.len(), 2);
assert_eq!(value[0].as_ref().unwrap(), &1);
let err = value[1].as_ref().unwrap_err();
assert!(
matches!(
err.kind(),
OperationErrorKind::ChildContext(c) if matches!(c.kind(), ChildContextErrorKind::ChildFailed)
),
"expected ChildContext(ChildFailed), got {:?}",
err.kind()
);
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[tokio::test]
async fn race_replay_children_drives_only_recorded_winner() {
let (ctx, client) = replay_children_ctx(Some("{\"winner\":1}"));
let flag = Arc::new(AtomicBool::new(false));
let op_id = ctx.mint_id();
let value = RaceExecution {
ctx,
op_id,
name: None,
futures: vec![flagging(&flag, 10), ready(20)],
}
.execute()
.await
.unwrap();
assert_eq!(value, 20);
assert!(!flag.load(Ordering::SeqCst), "input 0 must not run");
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[tokio::test]
async fn select_ok_replay_children_drives_only_recorded_winner() {
let (ctx, client) = replay_children_ctx(Some("{\"winner\":1}"));
let flag = Arc::new(AtomicBool::new(false));
let op_id = ctx.mint_id();
let value = SelectOkExecution {
ctx,
op_id,
name: None,
futures: vec![flagging(&flag, 10), ready(20)],
}
.execute()
.await
.unwrap();
assert_eq!(value, 20);
assert!(!flag.load(Ordering::SeqCst), "input 0 must not run");
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
async fn race_replay_children_error(payload: Option<&str>) -> String {
let (ctx, client) = replay_children_ctx(payload);
let op_id = ctx.mint_id();
let err = RaceExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![must_not_run(), must_not_run()],
}
.execute()
.await
.unwrap_err();
internal_message(&err);
assert_eq!(client.recorded_updates().len(), 0, "no update");
format!("{err:#}")
}
#[tokio::test]
async fn race_replay_children_rejects_out_of_range_winner() {
let text = race_replay_children_error(Some("{\"winner\":5}")).await;
assert!(
text.contains("recorded winner index 5 is out of range for 2 inputs"),
"{text}"
);
}
#[tokio::test]
async fn race_replay_children_rejects_malformed_winner_record() {
let text = race_replay_children_error(Some("{}")).await;
assert!(text.contains("malformed winner record"), "{text}");
}
#[tokio::test]
async fn race_replay_children_rejects_missing_winner_record() {
let text = race_replay_children_error(None).await;
assert!(text.contains("malformed winner record"), "{text}");
}
#[tokio::test]
async fn race_replay_children_returns_winner_error_unchanged() {
let (ctx, client) = replay_children_ctx(Some("{\"winner\":0}"));
let op_id = ctx.mint_id();
let err = RaceExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![failing_input(), must_not_run()],
}
.execute()
.await
.unwrap_err();
assert_eq!(internal_message(&err), "input failed");
assert_eq!(
err.wire().and_then(crate::error::WireError::error_type),
Some("InputFailure")
);
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[tokio::test]
async fn try_join_all_replay_without_replay_children_decodes_payload() {
let (wire_id, record) = succeeded_record("1", "[1,2]");
assert!(!record.replay_children);
let (ctx, client) = recording_ctx(CheckpointLog::from_records(vec![(wire_id, record)]));
let op_id = ctx.mint_id();
let value = TryJoinAllExecution::<i32> {
ctx,
op_id,
name: None,
futures: vec![must_not_run(), must_not_run()],
}
.execute()
.await
.unwrap();
assert_eq!(value, vec![1, 2]);
assert_eq!(client.recorded_updates().len(), 0, "no update");
}
#[derive(Clone, Copy, Debug)]
enum Outer {
TryJoinAll,
JoinAll,
SelectOk,
Race,
}
#[derive(Clone, Copy, Debug)]
enum Recorded {
Nothing,
Started,
Pending,
ReplayChildren,
}
#[tokio::test]
async fn unclaimed_input_check_covers_every_spawn_path() {
let outers = [
Outer::TryJoinAll,
Outer::JoinAll,
Outer::SelectOk,
Outer::Race,
];
let records = [
Recorded::Nothing,
Recorded::Started,
Recorded::Pending,
Recorded::ReplayChildren,
];
for outer in outers {
for recorded in records {
let log = match recorded {
Recorded::Nothing => CheckpointLog::empty(),
Recorded::Started | Recorded::Pending => {
let (wire_id, mut record) = succeeded_record("1", "");
record.result = None;
record.status = if matches!(recorded, Recorded::Started) {
CheckpointStatus::Started
} else {
CheckpointStatus::Pending
};
CheckpointLog::from_records(vec![(wire_id, record)])
}
Recorded::ReplayChildren => {
let payload = match outer {
Outer::TryJoinAll | Outer::JoinAll => "",
Outer::SelectOk | Outer::Race => r#"{"winner":0}"#,
};
let (wire_id, mut record) = succeeded_record("1", payload);
record.replay_children = true;
CheckpointLog::from_records(vec![(wire_id, record)])
}
};
let (ctx, client) = recording_ctx(log);
let polled = Arc::new(AtomicBool::new(false));
let inner = ctx.race(vec![flagging(&polled, 1)]).name("inner").future();
let inputs = vec![flagging(&polled, 2), inner];
let failed = match outer {
Outer::TryJoinAll => ctx.try_join_all(inputs).await.is_err(),
Outer::JoinAll => ctx.join_all(inputs).await.is_err(),
Outer::SelectOk => ctx.select_ok(inputs).await.is_err(),
Outer::Race => ctx.race(inputs).await.is_err(),
};
let case = format!("{outer:?} with {recorded:?}");
assert!(failed, "{case}: the outer combinator fails");
let fatal = ctx.coordinator().fatal_error().expect("a fatal error");
assert_eq!(fatal.error_type, "Error", "{case}");
assert_eq!(
fatal.error_message,
"context usage error in \"race\": the combinator was created on context \
\"root\", but combinator \"1\" started it as an input. Start the inner \
combinator with .spawn() and pass the returned handle to the outer \
combinator.",
"{case}"
);
assert!(!polled.load(Ordering::SeqCst), "{case}: no input is polled");
assert_eq!(
ctx.mint_id().positional(),
"2",
"{case}: the inner combinator claims nothing"
);
let actions: Vec<_> = client
.recorded_updates()
.iter()
.map(|u| u.action().clone())
.collect();
let expected = if matches!(recorded, Recorded::Nothing) {
vec![aws_sdk_lambda::types::OperationAction::Start]
} else {
vec![]
};
assert_eq!(actions, expected, "{case}: START only without a record");
}
}
}
const OUTERS: [Outer; 4] = [
Outer::TryJoinAll,
Outer::JoinAll,
Outer::SelectOk,
Outer::Race,
];
fn run_outer<O>(
ctx: &DurableContext,
outer: Outer,
inputs: Vec<DurableFuture<O>>,
) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send>>
where
O: Serialize + DeserializeOwned + Send + 'static,
{
use std::future::IntoFuture;
match outer {
Outer::TryJoinAll => {
let f = ctx.try_join_all(inputs).name("all").into_future();
Box::pin(async move { drop(f.await) })
}
Outer::JoinAll => {
let f = ctx.join_all(inputs).name("all").into_future();
Box::pin(async move { drop(f.await) })
}
Outer::SelectOk => {
let f = ctx.select_ok(inputs).name("all").into_future();
Box::pin(async move { drop(f.await) })
}
Outer::Race => {
let f = ctx.race(inputs).name("all").into_future();
Box::pin(async move { drop(f.await) })
}
}
}
fn spawn_outer<O>(
ctx: &DurableContext,
outer: Outer,
inputs: Vec<DurableFuture<O>>,
) -> std::pin::Pin<Box<dyn Future<Output = bool> + Send>>
where
O: Serialize + DeserializeOwned + Send + 'static,
{
match outer {
Outer::TryJoinAll => {
let h = ctx.try_join_all(inputs).name("all").spawn();
Box::pin(async move { h.await.is_ok() })
}
Outer::JoinAll => {
let h = ctx.join_all(inputs).name("all").spawn();
Box::pin(async move { h.await.is_ok() })
}
Outer::SelectOk => {
let h = ctx.select_ok(inputs).name("all").spawn();
Box::pin(async move { h.await.is_ok() })
}
Outer::Race => {
let h = ctx.race(inputs).name("all").spawn();
Box::pin(async move { h.await.is_ok() })
}
}
}
async fn within_two_seconds(mut ready: impl FnMut() -> bool) -> bool {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
while !ready() {
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
}
true
}
type Submitted = (
String,
String,
aws_sdk_lambda::types::OperationAction,
Arc<AtomicBool>,
);
fn record_submissions(ctx: &DurableContext) -> Arc<std::sync::Mutex<Vec<Submitted>>> {
let submissions: Arc<std::sync::Mutex<Vec<Submitted>>> = Arc::default();
let recorded = Arc::clone(&submissions);
ctx.set_submission_recorder(Arc::new(move |updates, flag| {
let mut recorded = recorded.lock().unwrap();
for u in updates {
recorded.push((
u.name().unwrap_or("-").to_owned(),
u.id().to_owned(),
u.action().clone(),
Arc::clone(flag),
));
}
}));
submissions
}
#[derive(Clone, Copy, Debug)]
enum Written {
Succeed,
Fail,
Empty,
}
impl Written {
fn action(self) -> aws_sdk_lambda::types::OperationAction {
match self {
Self::Succeed => aws_sdk_lambda::types::OperationAction::Succeed,
Self::Fail | Self::Empty => aws_sdk_lambda::types::OperationAction::Fail,
}
}
fn inputs(self) -> Vec<DurableFuture<i32>> {
match self {
Self::Succeed => vec![ready(1), ready(2)],
Self::Fail => vec![failing_input(), failing_input()],
Self::Empty => Vec::new(),
}
}
}
fn outcome_cases() -> Vec<(Outer, Written)> {
let mut cases: Vec<(Outer, Written)> =
OUTERS.iter().map(|o| (*o, Written::Succeed)).collect();
cases.extend(
[Outer::TryJoinAll, Outer::SelectOk, Outer::Race]
.into_iter()
.map(|o| (o, Written::Fail)),
);
cases.extend(
[Outer::SelectOk, Outer::Race]
.into_iter()
.map(|o| (o, Written::Empty)),
);
cases
}
async fn outcome_case(outer: Outer, written: Written, fail_start: bool) -> Result<(), String> {
use aws_sdk_lambda::types::OperationAction;
let (ctx, client) = recording_ctx(CheckpointLog::empty());
if fail_start {
client.enqueue_checkpoint_response(crate::client::TestResponse::NonRetryableError(
"START rejected".to_owned(),
));
}
let gate = Arc::new(tokio::sync::Notify::new());
let held = Arc::clone(&gate);
ctx.set_pre_lock_gate(Arc::new(move |updates| {
updates
.iter()
.any(|u| u.action() == &OperationAction::Start && u.name() == Some("all"))
.then(|| Arc::clone(&held))
}));
let submissions = record_submissions(&ctx);
let entries: Arc<std::sync::Mutex<Vec<(String, OperationAction)>>> = Arc::default();
let entered = Arc::clone(&entries);
ctx.set_settler_entry_recorder(Arc::new(move |wire_id, action| {
entered.lock().unwrap().push((wire_id.to_owned(), action));
}));
let wire = crate::engine::compute_wire_id_public("1");
let action = written.action();
let outcomes = |submissions: &[Submitted]| {
submissions
.iter()
.filter(|(_, id, a, _)| {
*id == wire && matches!(a, OperationAction::Succeed | OperationAction::Fail)
})
.count()
};
let mut combinator = run_outer(&ctx, outer, written.inputs());
let marker = within_two_seconds(|| {
entries
.lock()
.unwrap()
.contains(&(wire.clone(), action.clone()))
});
tokio::select! {
() = &mut combinator => return Err("resolved before the settler-entry marker".to_owned()),
reached = marker => if !reached {
return Err("no settler-entry marker within two seconds".to_owned());
},
}
let start_polled = submissions
.lock()
.unwrap()
.iter()
.find(|(name, _, a, _)| name == "all" && a == &OperationAction::Start)
.is_some_and(|(_, _, _, flag)| flag.load(Ordering::SeqCst));
if !start_polled {
return Err("the retained START was not polled".to_owned());
}
if outcomes(&submissions.lock().unwrap()) != 0 {
return Err("an outcome was submitted while START was held".to_owned());
}
gate.notify_one();
if fail_start {
if tokio::time::timeout(std::time::Duration::from_millis(500), &mut combinator)
.await
.is_ok()
{
return Err("resolved after a failed START".to_owned());
}
let fatal = ctx
.coordinator()
.fatal_error()
.ok_or_else(|| "no fatal error".to_owned())?;
let expected = crate::error::checkpoint_execution_failure_wire(
&crate::client::ExecutionClientError::non_retryable("START rejected".to_owned()),
);
if fatal.error_type != "CheckpointUnrecoverableExecutionError"
|| Some(fatal.error_message.as_str()) != expected.error_message()
{
return Err(format!("fatal {fatal:?}"));
}
if outcomes(&submissions.lock().unwrap()) != 0 {
return Err("an outcome was submitted after a failed START".to_owned());
}
} else {
tokio::time::timeout(std::time::Duration::from_secs(2), &mut combinator)
.await
.map_err(|_| "did not resolve after the release".to_owned())?;
if outcomes(&submissions.lock().unwrap()) != 1 {
return Err("no outcome after the release".to_owned());
}
let recorded: Vec<OperationAction> = client
.recorded_updates()
.iter()
.filter(|u| u.id() == wire)
.map(|u| u.action().clone())
.collect();
if recorded != [OperationAction::Start, action] {
return Err(format!("recorded {recorded:?}"));
}
}
Ok(())
}
#[tokio::test]
async fn combinator_outcome_waits_for_retained_start_write() {
let mut failures = Vec::new();
for (outer, written) in outcome_cases() {
for fail_start in [false, true] {
if let Err(err) = outcome_case(outer, written, fail_start).await {
failures.push(format!(
"{outer:?} {written:?} fail_start={fail_start}: {err}"
));
}
}
}
assert!(failures.is_empty(), "failed cases: {failures:#?}");
}
fn record_of(
positional: &str,
status: CheckpointStatus,
op_type: &str,
sub_type: &str,
name: &str,
) -> (String, CheckpointRecord) {
let (wire_id, mut record) = succeeded_record(positional, "");
record.result = None;
record.status = status;
record.op_type = Some(op_type.to_owned());
record.sub_type = Some(sub_type.to_owned());
record.op_name = Some(name.to_owned());
(wire_id, record)
}
#[tokio::test]
async fn live_join_info_is_the_combinator_info() {
use crate::plugin::recorder::{UfHooks, context};
let join = |ctx: &DurableContext| {
let a = ctx.step(|_| async { Ok(1_u32) }).name("a").future();
let b = ctx.step(|_| async { Ok(2_u32) }).name("b").future();
ctx.try_join_all([a, b]).name("all")
};
for (run, records) in [
(1, Vec::new()),
(
2,
vec![record_of(
"3",
CheckpointStatus::Started,
"Context",
COMBINATOR_SUB_TYPE,
"all",
)],
),
] {
let hooks = UfHooks::default();
let (ctx, _client) = context(&hooks, records);
assert_eq!(join(&ctx).await.unwrap(), [1, 2], "run {run}");
let start = hooks.find("all", "start");
assert_eq!(start.replay, run == 2, "run {run}");
assert_eq!(start.id, crate::engine::compute_wire_id_public("3"));
let join = hooks.user_function_of("all", true);
for record in &join {
assert_eq!(record.replay, run == 2, "run {run}: {}", record.hook);
assert_eq!(
record.info, start.info,
"run {run}: {} gets the combinator's info",
record.hook
);
}
}
}
type TypeAndMessage = (Option<String>, Option<String>);
fn join_end(
hooks: &crate::plugin::recorder::UfHooks,
client: &InMemoryExecutionClient,
name: &str,
) -> Option<(TypeAndMessage, TypeAndMessage)> {
use crate::plugin::recorder::{type_and_message, update_error, update_of};
let end = hooks.find(name, "uf_end");
let wire = end.wire.as_ref()?;
let fail = update_of(
&client.recorded_updates(),
name,
&aws_sdk_lambda::types::OperationAction::Fail,
)
.expect("the join's FAIL");
Some((type_and_message(wire), update_error(&fail)))
}
#[tokio::test]
async fn combinator_join_outcomes() {
use crate::plugin::recorder::{UfHooks, context, update_of};
let ok = |ctx: &DurableContext| ctx.step(|_| async { Ok(1_u32) }).name("ok").future();
let bad = |ctx: &DurableContext| {
ctx.step(|_| async { Err::<u32, crate::BoxError>("boom".into()) })
.name("bad")
.retry_strategy(|_err, _attempt| crate::RetryDecision::Stop)
.future()
};
let failing = |expected: &str, hooks: &UfHooks, client: &InMemoryExecutionClient| {
hooks.user_function_of("j", true);
let (end, stored) = join_end(hooks, client, "j").expect("an Err outcome");
assert_eq!(end.0.as_deref(), Some(expected), "{end:?}");
assert_eq!(
end, stored,
"{expected}: the FAIL stores the end hook's wire"
);
};
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let inputs = [ok(&ctx), bad(&ctx)];
drop(ctx.try_join_all(inputs).name("j").await.unwrap_err());
failing("CombinatorError.JoinFailed.1", &hooks, &client);
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let inputs = [ok(&ctx), bad(&ctx)];
let outcomes = ctx.join_all(inputs).name("j").await.unwrap();
assert!(outcomes.first().is_some_and(Result::is_ok));
let item = outcomes
.get(1)
.and_then(|o| o.as_ref().err())
.expect("input 1 failed");
let item = crate::plugin::recorder::type_and_message(item.wire().expect("a wire"));
assert_eq!(item, (Some("Error".to_owned()), Some("boom".to_owned())));
hooks.user_function_of("j", true);
assert_eq!(hooks.find("j", "uf_end").ok, Some(true));
let succeed = update_of(
&client.recorded_updates(),
"j",
&aws_sdk_lambda::types::OperationAction::Succeed,
);
assert!(succeed.is_some(), "join_all records a SUCCEED");
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let inputs = [bad(&ctx)];
drop(ctx.select_ok(inputs).name("j").await.unwrap_err());
failing(ALL_FAILED_ERROR_TYPE, &hooks, &client);
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let inputs = [bad(&ctx)];
drop(ctx.race(inputs).name("j").await.unwrap_err());
failing(FIRST_SETTLED_FAILED_ERROR_TYPE, &hooks, &client);
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let none: [DurableFuture<u32>; 0] = [];
drop(ctx.select_ok(none).name("j").await.unwrap_err());
failing(EMPTY_INPUT_ERROR_TYPE, &hooks, &client);
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let none: [DurableFuture<u32>; 0] = [];
drop(ctx.race(none).name("j").await.unwrap_err());
failing(EMPTY_INPUT_ERROR_TYPE, &hooks, &client);
let hooks = UfHooks::default();
let (ctx, _client) = context(&hooks, Vec::new());
let started = Arc::new(tokio::sync::Notify::new());
let waits = Arc::clone(&started);
let winner = ctx
.step(move |_| async move {
waits.notified().await;
Ok(1_u32)
})
.name("winner")
.future();
let loser = ctx
.step(move |_| async move {
started.notify_one();
std::future::pending::<()>().await;
Ok(2_u32)
})
.name("loser")
.future();
let value = tokio::time::timeout(
std::time::Duration::from_secs(5),
ctx.race([winner, loser]).name("j").into_future(),
)
.await
.expect("the race ends within 5 s")
.unwrap();
assert_eq!(value, 1);
hooks.user_function_of("loser", false);
hooks.user_function_of("winner", true);
hooks.user_function_of("j", true);
assert_eq!(hooks.find("j", "uf_end").ok, Some(true));
}
#[tokio::test]
async fn empty_collection_joins_fire_join_hooks() {
use crate::plugin::recorder::{UfHooks, context, update_of};
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let none: [DurableFuture<u32>; 0] = [];
assert_eq!(
ctx.try_join_all(none).name("t").await.unwrap(),
Vec::<u32>::new()
);
let none: [DurableFuture<u32>; 0] = [];
assert!(ctx.join_all(none).name("j").await.unwrap().is_empty());
let updates = client.recorded_updates();
for name in ["t", "j"] {
hooks.user_function_of(name, true);
assert_eq!(hooks.find(name, "uf_end").ok, Some(true), "{name}");
let succeed = update_of(
&updates,
name,
&aws_sdk_lambda::types::OperationAction::Succeed,
)
.expect("the join's SUCCEED");
assert_eq!(succeed.payload(), Some("[]"), "{name}");
}
}
fn plugin_ctx(
log: CheckpointLog,
window: Option<std::time::Duration>,
plugins: Arc<crate::plugin::InvocationPlugins>,
) -> (DurableContext, Arc<InMemoryExecutionClient>) {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token-1".to_owned(),
window,
0,
Some(plugins),
);
(ctx, client)
}
#[derive(Clone)]
struct TraceSpan;
impl crate::plugin::Plugin for TraceSpan {
fn wrap_user_function(&self, info: &crate::plugin::OperationInfo, next: &mut dyn FnMut()) {
let span = tracing::info_span!("plugin_user_function", trace_id = %info.id());
span.in_scope(next);
}
}
impl crate::plugin::PluginFactory for TraceSpan {
fn create_plugin(
&self,
_info: &crate::plugin::InvocationInfo,
) -> Result<Option<Box<dyn crate::plugin::Plugin>>, crate::BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
#[tokio::test]
async fn run_join_body_record_is_under_the_plugin_span() {
use tracing::Instrument as _;
let (buffer, _guard) = crate::plugin::test_support::capture();
let (ctx, _client) = plugin_ctx(
CheckpointLog::empty(),
None,
crate::plugin::test_support::plugins_of(TraceSpan),
);
let caller = ctx.replay_span().in_scope(|| {
crate::tracing_layer::scoped_execution_span(
"arn:aws:lambda:us-east-1:1:function:test",
"req-caller",
None,
"caller-op",
Some("caller-name"),
false,
)
});
let joined = crate::context::run_body(
crate::context::ActiveBody::Context("caller".to_owned()),
run_join(
&ctx,
"7",
"join-wire",
Some("j"),
Run::Live,
async {
tracing::info!("join body");
Ok::<_, std::convert::Infallible>(crate::context::current_body())
},
crate::plugin::never_fails,
),
)
.instrument(caller)
.await;
let Ok(active) = joined;
assert!(
matches!(&active, crate::context::ActiveBody::Context(id) if id == "caller"),
"{active:?}"
);
let output = String::from_utf8_lossy(&buffer.lock().unwrap()).to_string();
let record = output
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|v| v["message"] == "join body")
.unwrap_or_else(|| panic!("no join body record in {output}"));
let spans = record["spans"].as_array().expect("a span list");
let names: Vec<&str> = spans.iter().filter_map(|s| s["name"].as_str()).collect();
assert_eq!(
names,
[
"durable_execution",
"durable_execution",
"plugin_user_function"
],
"{record}"
);
assert_eq!(spans[2]["trace_id"], "join-wire", "{record}");
assert_eq!(spans[1]["operationId"], "caller-op", "{record}");
assert_eq!(spans[1]["operationName"], "caller-name", "{record}");
assert_eq!(spans[1]["requestId"], "req-caller", "{record}");
assert!(spans[0].get("operationId").is_none(), "{record}");
assert!(spans[2].get("operationId").is_none(), "{record}");
assert_eq!(record.get("operationId"), None, "{record}");
}
#[tokio::test]
async fn resumable_combinator_inputs_start_before_combinator_replay_hook() {
let mut failures = Vec::new();
for status in [CheckpointStatus::Pending, CheckpointStatus::Ready] {
for outer in OUTERS {
let hooks = crate::plugin::test_support::OpHooks::default();
let log = CheckpointLog::from_records(vec![
record_of("1", CheckpointStatus::Started, "Wait", "Wait", "a"),
record_of("2", CheckpointStatus::Started, "Wait", "Wait", "b"),
record_of("3", status.clone(), "Context", COMBINATOR_SUB_TYPE, "all"),
]);
let (ctx, _client) = plugin_ctx(log, None, hooks.plugins());
let a = ctx
.wait(std::time::Duration::from_secs(1))
.name("a")
.future();
let b = ctx
.wait(std::time::Duration::from_secs(2))
.name("b")
.future();
let replays = || -> Vec<String> {
hooks
.records()
.into_iter()
.filter(|r| r.hook == "start" && r.replay)
.filter_map(|r| r.name)
.collect()
};
let combinator = run_outer(&ctx, outer, vec![a, b]);
tokio::select! {
() = combinator => {}
_ = within_two_seconds(|| replays().len() >= 3) => {}
}
let replays = replays();
if replays != ["a", "b", "all"] {
failures.push(format!("{outer:?} {status:?}: {replays:?}"));
}
}
}
assert!(failures.is_empty(), "failed cases: {failures:#?}");
}
#[tokio::test(flavor = "current_thread")]
async fn spawned_combinator_starts_inputs_on_its_task_in_order() {
let mut failures = Vec::new();
for outer in OUTERS {
let (ctx, _client) = recording_ctx(CheckpointLog::empty());
let submissions = record_submissions(&ctx);
let starts = || -> Vec<String> {
submissions
.lock()
.unwrap()
.iter()
.filter(|(_, _, a, _)| a == &aws_sdk_lambda::types::OperationAction::Start)
.map(|(name, _, _, _)| name.clone())
.collect()
};
let a = ctx.step(|_| async { Ok(1_i32) }).name("a").future();
let b = ctx.step(|_| async { Ok(2_i32) }).name("b").future();
let handle = spawn_outer(&ctx, outer, vec![a, b]);
let at_spawn = starts();
for _ in 0..10 {
if starts().len() >= 3 {
break;
}
tokio::task::yield_now().await;
}
let after = starts();
if !at_spawn.is_empty() || after != ["a", "b", "all"] {
failures.push(format!("{outer:?}: at spawn {at_spawn:?}, after {after:?}"));
}
let _succeeded = handle.await;
}
assert!(failures.is_empty(), "failed cases: {failures:#?}");
}
#[derive(Default)]
struct HookGate {
entered: bool,
released: bool,
}
#[derive(Clone, Default)]
struct LogHooks {
log: Arc<std::sync::Mutex<Vec<String>>>,
gate: Option<Arc<(std::sync::Mutex<HookGate>, std::sync::Condvar)>>,
}
impl LogHooks {
fn entries(&self) -> Vec<String> {
self.log.lock().unwrap().clone()
}
fn position(&self, entry: &str) -> Option<usize> {
self.entries().iter().position(|e| e == entry)
}
fn release(&self) {
if let Some(gate) = &self.gate {
gate.0.lock().unwrap().released = true;
gate.1.notify_all();
}
}
fn entered(&self) -> bool {
self.gate
.as_ref()
.is_some_and(|gate| gate.0.lock().unwrap().entered)
}
}
impl crate::plugin::Plugin for LogHooks {
fn on_operation_start(
&self,
info: &crate::plugin::OperationInfo,
) -> crate::plugin::HookWork {
let name = info.name().unwrap_or("-");
self.log.lock().unwrap().push(format!("hook {name}"));
if name == "all"
&& let Some(gate) = &self.gate
{
tokio::task::block_in_place(|| {
let (state, condvar) = &**gate;
let mut state = state.lock().unwrap();
state.entered = true;
let _unused = condvar
.wait_timeout_while(state, std::time::Duration::from_secs(5), |s| {
!s.released
})
.unwrap();
});
}
self.log.lock().unwrap().push(format!("hook-return {name}"));
Ok(None)
}
}
impl crate::plugin::PluginFactory for LogHooks {
fn create_plugin(
&self,
_info: &crate::plugin::InvocationInfo,
) -> Result<Option<Box<dyn crate::plugin::Plugin>>, crate::BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn combinator_queue_then_hook_blocks_start_send_until_hook_dispatch() {
let mut failures = Vec::new();
for outer in OUTERS {
for window in [None, Some(std::time::Duration::ZERO)] {
let hooks = LogHooks {
gate: Some(Arc::default()),
..LogHooks::default()
};
let (ctx, client) = plugin_ctx(
CheckpointLog::empty(),
window,
crate::plugin::test_support::plugins_of(hooks.clone()),
);
let sent = || {
client.recorded_updates().iter().any(|u| {
u.action() == &aws_sdk_lambda::types::OperationAction::Start
&& u.name() == Some("all")
})
};
let handle = spawn_outer(&ctx, outer, vec![ready(1_i32), ready(2_i32)]);
let entered = within_two_seconds(|| hooks.entered()).await;
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let sent_while_blocked = sent();
hooks.release();
let succeeded = tokio::time::timeout(std::time::Duration::from_secs(5), handle)
.await
.unwrap_or(false);
let sent_after = sent();
if !entered || sent_while_blocked || !sent_after || !succeeded {
failures.push(format!(
"{outer:?} window={window:?}: entered={entered} \
sent_while_blocked={sent_while_blocked} sent_after={sent_after} \
succeeded={succeeded}"
));
}
}
}
assert!(failures.is_empty(), "failed cases: {failures:#?}");
}
#[derive(Clone, Copy, Debug)]
enum ResponseFirst {
Wait,
AtMostOnceStep,
}
impl ResponseFirst {
fn input(self, ctx: &DurableContext, name: &str) -> DurableFuture<()> {
match self {
Self::Wait => ctx
.wait(std::time::Duration::from_secs(1))
.name(name)
.future(),
Self::AtMostOnceStep => ctx
.step(|_| std::future::pending::<Result<(), crate::BoxError>>())
.name(name)
.semantics(crate::StepSemantics::AtMostOncePerRetry)
.future(),
}
}
}
#[tokio::test(flavor = "current_thread")]
async fn response_first_remainders_start_after_combinator_start_hook() {
let mut failures = Vec::new();
for outer in OUTERS {
for kind in [ResponseFirst::Wait, ResponseFirst::AtMostOnceStep] {
let hooks = LogHooks::default();
let (ctx, _client) = plugin_ctx(
CheckpointLog::empty(),
None,
crate::plugin::test_support::plugins_of(hooks.clone()),
);
let log = Arc::clone(&hooks.log);
REMAINDER_RECORDER.with(|recorder| {
*recorder.borrow_mut() = Some(Box::new(move |idx| {
let name = if idx == 0 { "a" } else { "b" };
log.lock().unwrap().push(format!("remainder {name}"));
}));
});
let inputs = vec![kind.input(&ctx, "a"), kind.input(&ctx, "b")];
let combinator = run_outer(&ctx, outer, inputs);
tokio::select! {
() = combinator => {}
_ = within_two_seconds(|| {
hooks.position("hook a").is_some() && hooks.position("hook b").is_some()
}) => {}
}
REMAINDER_RECORDER.with(|recorder| *recorder.borrow_mut() = None);
let at = |entry: &str| hooks.position(entry);
let ordered = ["hook all", "hook-return all", "remainder a", "remainder b"]
.windows(2)
.all(|pair| matches!((at(pair[0]), at(pair[1])), (Some(x), Some(y)) if x < y));
let inputs_after = ["hook a", "hook b"].iter().all(|entry| {
matches!((at("hook-return all"), at(entry)), (Some(x), Some(y)) if x < y)
});
if !ordered || !inputs_after {
failures.push(format!("{outer:?} {kind:?}: {:?}", hooks.entries()));
}
}
}
assert!(failures.is_empty(), "failed cases: {failures:#?}");
}
fn loser_with_message(message: &str) -> crate::error::Source {
Box::new(
OperationError::from_kind(OperationErrorKind::Combinator(CombinatorError::new(
CombinatorErrorKind::Internal,
vec![format!("{message} source").into()],
)))
.with_wire(crate::error::WireError::new(Some("Error"), Some(message))),
)
}
#[test]
fn all_failed_message_names_each_loser_message() {
let wire = all_failed_wire(&[loser_with_message("a"), loser_with_message("b")]);
assert_eq!(
wire.error_message(),
Some("all 2 futures failed: [0] a; [1] b")
);
}
#[test]
fn all_failed_typed_losers_record_the_wrapped_message() {
let plain: crate::error::Source =
Box::new(crate::TypedError::with_type("PaymentError", "declined").with_error_data("d"));
let propagated: crate::error::Source = Box::new(crate::TypedError::with_type(
"PaymentError",
crate::error::recorded_step_error(crate::error::WireError::new(
Some("InnerType"),
Some("boom"),
)),
));
let wire = all_failed_wire(&[plain, propagated]);
assert_eq!(
wire.error_message(),
Some("all 2 futures failed: [0] declined; [1] boom")
);
let losers: Vec<SerializedLoser> =
serde_json::from_str(wire.error_data().expect("the losers")).expect("loser JSON");
let losers: Vec<crate::error::WireError> =
losers.into_iter().map(SerializedLoser::into_wire).collect();
assert_eq!(
losers,
[
crate::error::WireError::new(Some("PaymentError"), Some("declined"))
.with_error_data(Some("d")),
crate::error::WireError::new(Some("PaymentError"), Some("boom")),
]
);
}
#[test]
fn all_failed_preserves_serialized_loser_message() {
let inner = crate::error::WireError::new(Some("InnerType"), Some("a"))
.with_error_data(Some("loser-data"))
.with_stack_trace(vec!["loser-frame".to_owned()]);
let loser = crate::error::ContextualError::source_from(
"context",
crate::error::recorded_step_error(inner),
);
let wire = all_failed_wire(&[loser]);
assert_eq!(wire.error_message(), Some("all 1 futures failed: [0] a"));
assert_eq!(
wire.error_data(),
Some(
r#"[{"error_type":"CombinatorError","message":"a","error_data":"loser-data","stack_trace":["loser-frame"]}]"#
)
);
}
#[tokio::test]
async fn join_all_rejected_outcome_records_the_propagated_message() {
let (ctx, _client) = recording_ctx(CheckpointLog::empty());
let input = ctx
.run_in_child_context(|child| async move {
Ok(child
.step(|_| async { Err::<u32, crate::BoxError>("a".into()) })
.retry_strategy(|_err, _attempt| crate::RetryDecision::Stop)
.await?)
})
.virtual_context()
.future();
let outcomes = ctx.join_all([input]).await.expect("join_all succeeds");
let [Err(rejected)] = outcomes.as_slice() else {
panic!("expected one rejected outcome, got {outcomes:?}");
};
let wire = rejected
.wire()
.expect("a rejected outcome carries its record");
assert_eq!(wire.error_type(), Some("ChildContextError"));
assert_eq!(wire.error_message(), Some("a"));
}
}
#[cfg(test)]
mod coordinator_tests {
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{
ScriptedClient, drive_handler, named, operation_like, scripted_ctx, wire_err,
};
use aws_sdk_lambda::types::{OperationAction, OperationStatus, OperationType};
use std::sync::Arc;
use std::time::Duration;
#[tokio::test(start_paused = true)]
async fn dropped_race_loser_retires_its_registration() {
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let outcome = drive_handler(ctx, None, |ctx| async move {
let fast = ctx
.step(|_| async {
tokio::time::sleep(Duration::from_millis(10)).await;
Ok(1_u32)
})
.name("fast")
.future();
let slow = ctx.wait(Duration::from_secs(3600)).name("slow").future();
let slow = crate::DurableFuture::from_async(async move {
slow.await?;
Ok(0_u32)
});
ctx.race([fast, slow]).await.map_err(wire_err)?;
ctx.hold_activity(ctx.coordinator().tasks().wait_idle())
.await;
let waits = ctx.coordinator().live_waits();
ctx.step(|_| async { Ok(2_u32) })
.name("more")
.await
.map_err(wire_err)?;
Ok(waits.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("0".to_owned()));
}
#[tokio::test(start_paused = true)]
async fn race_with_every_input_waiting_suspends() {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let one = ctx.wait(Duration::from_secs(1)).future();
let two = ctx.wait(Duration::from_secs(2)).future();
ctx.race([one, two]).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Pending);
let calls = client.calls();
let starts: Vec<OperationType> = calls
.iter()
.flat_map(|(_, updates)| updates.iter())
.inspect(|u| assert_eq!(u.action(), &OperationAction::Start))
.map(|u| u.r#type().clone())
.collect();
assert_eq!(
starts,
[
OperationType::Wait,
OperationType::Wait,
OperationType::Context
],
"the two wait STARTs, then the race START, and no later call"
);
}
#[tokio::test(start_paused = true)]
async fn waiting_input_continues_on_sibling_response() {
let client = Arc::new(ScriptedClient::new(|updates, history| {
if named(updates, "s", &OperationAction::Succeed).is_none() {
return Vec::new();
}
named(history, "w", &OperationAction::Start)
.map(|start| {
operation_like(start, OperationStatus::Succeeded)
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
}));
let ctx = scripted_ctx(&client);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let child = ctx
.run_in_child_context(|child| async move {
child.wait(Duration::from_secs(1)).name("w").await?;
Ok(1_u32)
})
.name("child")
.future();
let step = ctx
.step(move |_| async move {
for _ in 0..1000 {
if body_client.received("w", &OperationAction::Start) {
return Ok(2_u32);
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
Err("w never started".into())
})
.name("s")
.future();
let values = ctx.try_join_all([child, step]).await.map_err(wire_err)?;
Ok(format!("{values:?}"))
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("[1, 2]".to_owned()));
}
}