use std::future::Future;
use std::sync::Arc;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use tracing::Instrument as _;
use crate::BoxError;
use crate::Serdes;
use crate::checkpoint_coalescer::ScopeClose;
use crate::child::CHECKPOINT_SIZE_LIMIT_BYTES;
use crate::context::{DurableContext, SubmittedWrite, run_body};
use crate::engine::{CheckpointStatus, OperationId};
use crate::error::{
ChildContextError, ChildContextErrorKind, ChildFnError, OperationError, OperationErrorKind,
};
use crate::plugin::UserBody;
use crate::serdes::SerdesContext;
use crate::start::StartStep;
use crate::task::{NoRuntime, TaskOutcome, TrackedJoinSet};
use serde_json::value::RawValue;
pub(crate) const MAP_SUB_TYPE: &str = "Map";
pub(crate) const MAP_ITERATION_SUB_TYPE: &str = "MapIteration";
pub(crate) const PARALLEL_SUB_TYPE: &str = "Parallel";
pub(crate) const PARALLEL_BRANCH_SUB_TYPE: &str = "ParallelBranch";
const REPLAY_CHILDREN_SENTINEL: &str = "__replay_children_reexecute__";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BatchItemStatus {
Succeeded,
Failed,
Started,
}
#[derive(Debug)]
#[non_exhaustive]
pub struct BatchItem<O> {
pub index: usize,
pub name: String,
pub status: BatchItemStatus,
pub result: Option<O>,
pub error_message: Option<String>,
pub error_type: Option<String>,
pub error_data: Option<String>,
pub stack_trace: Vec<String>,
}
impl<O> BatchItem<O> {
#[must_use]
pub fn new(
index: usize,
name: String,
status: BatchItemStatus,
result: Option<O>,
error_message: Option<String>,
error_type: Option<String>,
) -> Self {
Self {
index,
name,
status,
result,
error_message,
error_type,
error_data: None,
stack_trace: Vec::new(),
}
}
pub(crate) fn failed_from_wire(
index: usize,
name: String,
wire: &crate::error::WireError,
) -> Self {
Self {
index,
name,
status: BatchItemStatus::Failed,
result: None,
error_message: wire.error_message().map(str::to_owned),
error_type: wire.error_type().map(str::to_owned),
error_data: wire.error_data().map(str::to_owned),
stack_trace: wire.stack_trace().to_vec(),
}
}
pub(crate) fn failure_wire(&self) -> crate::error::WireError {
crate::error::WireError::new(
self.error_type.as_deref(),
Some(
self.error_message
.as_deref()
.unwrap_or("branch failed")
.to_owned(),
),
)
.with_error_data(self.error_data.as_deref())
.with_stack_trace(self.stack_trace.clone())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BatchStatus {
Succeeded,
Failed,
}
impl BatchStatus {
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::Succeeded => "SUCCEEDED",
Self::Failed => "FAILED",
}
}
}
impl std::fmt::Display for BatchStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct BatchError<'a> {
pub index: usize,
pub name: &'a str,
pub message: &'a str,
pub error_type: Option<&'a str>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum CompletionReason {
AllCompleted,
MinSuccessfulReached,
FailureToleranceExceeded,
CustomCompletionSucceeded,
CustomCompletionFailed,
}
impl CompletionReason {
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::AllCompleted => "ALL_COMPLETED",
Self::MinSuccessfulReached => "MIN_SUCCESSFUL_REACHED",
Self::FailureToleranceExceeded => "FAILURE_TOLERANCE_EXCEEDED",
Self::CustomCompletionSucceeded => "CUSTOM_COMPLETION_SUCCEEDED",
Self::CustomCompletionFailed => "CUSTOM_COMPLETION_FAILED",
}
}
pub(crate) fn from_wire(s: &str) -> Self {
match s {
"MIN_SUCCESSFUL_REACHED" => Self::MinSuccessfulReached,
"FAILURE_TOLERANCE_EXCEEDED" => Self::FailureToleranceExceeded,
"CUSTOM_COMPLETION_SUCCEEDED" => Self::CustomCompletionSucceeded,
"CUSTOM_COMPLETION_FAILED" => Self::CustomCompletionFailed,
_ => Self::AllCompleted,
}
}
pub(crate) fn try_from_wire(s: &str) -> Option<Self> {
match s {
"ALL_COMPLETED" => Some(Self::AllCompleted),
"MIN_SUCCESSFUL_REACHED" => Some(Self::MinSuccessfulReached),
"FAILURE_TOLERANCE_EXCEEDED" => Some(Self::FailureToleranceExceeded),
"CUSTOM_COMPLETION_SUCCEEDED" => Some(Self::CustomCompletionSucceeded),
"CUSTOM_COMPLETION_FAILED" => Some(Self::CustomCompletionFailed),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum CompletionDecision {
Continue,
#[non_exhaustive]
Complete(CompletionOutcome),
}
impl CompletionDecision {
#[must_use]
pub fn complete(outcome: CompletionOutcome) -> Self {
Self::Complete(outcome)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum CompletionOutcome {
Succeeded,
Failed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct CompletionItem {
pub index: usize,
pub name: Option<String>,
pub status: Option<BatchItemStatus>,
}
impl CompletionItem {
#[must_use]
pub fn new(index: usize, name: Option<String>, status: Option<BatchItemStatus>) -> Self {
Self {
index,
name,
status,
}
}
}
#[derive(Debug, Clone, Copy)]
#[non_exhaustive]
pub struct BatchStats<'a> {
succeeded: usize,
failed: usize,
items: &'a [CompletionItem],
}
impl<'a> BatchStats<'a> {
#[must_use]
pub fn new(items: &'a [CompletionItem]) -> Self {
let count = |status| {
items
.iter()
.filter(|item| item.status == Some(status))
.count()
};
Self {
succeeded: count(BatchItemStatus::Succeeded),
failed: count(BatchItemStatus::Failed),
items,
}
}
pub(crate) fn from_counts(
succeeded: usize,
failed: usize,
items: &'a [CompletionItem],
) -> Self {
Self {
succeeded,
failed,
items,
}
}
pub fn succeeded(&self) -> usize {
self.succeeded
}
pub fn failed(&self) -> usize {
self.failed
}
pub fn settled(&self) -> usize {
self.succeeded + self.failed
}
pub fn total_items(&self) -> usize {
self.items.len()
}
pub fn items(&self) -> &'a [CompletionItem] {
self.items
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct BatchResult<O> {
pub items: Vec<BatchItem<O>>,
pub reason: CompletionReason,
}
impl<O> BatchResult<O> {
#[must_use]
pub fn new(items: Vec<BatchItem<O>>, reason: CompletionReason) -> Self {
Self { items, reason }
}
pub fn results(&self) -> impl Iterator<Item = &O> + '_ {
self.succeeded().filter_map(|item| item.result.as_ref())
}
pub fn success_count(&self) -> usize {
self.items
.iter()
.filter(|i| i.status == BatchItemStatus::Succeeded)
.count()
}
pub fn failure_count(&self) -> usize {
self.items
.iter()
.filter(|i| i.status == BatchItemStatus::Failed)
.count()
}
pub fn has_failure(&self) -> bool {
self.items
.iter()
.any(|i| i.status == BatchItemStatus::Failed)
}
pub fn status(&self) -> BatchStatus {
match self.reason {
CompletionReason::CustomCompletionFailed => BatchStatus::Failed,
CompletionReason::CustomCompletionSucceeded => BatchStatus::Succeeded,
_ if self.has_failure() => BatchStatus::Failed,
_ => BatchStatus::Succeeded,
}
}
pub fn total_count(&self) -> usize {
self.items.len()
}
pub fn errors(&self) -> impl Iterator<Item = BatchError<'_>> + '_ {
self.failed().map(|item| BatchError {
index: item.index,
name: &item.name,
message: item.error_message.as_deref().unwrap_or_default(),
error_type: item.error_type.as_deref(),
})
}
pub fn succeeded(&self) -> impl Iterator<Item = &BatchItem<O>> + '_ {
self.with_status(BatchItemStatus::Succeeded)
}
pub fn failed(&self) -> impl Iterator<Item = &BatchItem<O>> + '_ {
self.with_status(BatchItemStatus::Failed)
}
pub fn started(&self) -> impl Iterator<Item = &BatchItem<O>> + '_ {
self.with_status(BatchItemStatus::Started)
}
pub fn started_count(&self) -> usize {
self.started().count()
}
pub fn into_result(self) -> Result<Self, OperationError> {
if let Some(item) = self.failed().next() {
return Err(item_failure_error(item.failure_wire()));
}
if self.status() == BatchStatus::Failed {
return Err(batch_completion_failed_error());
}
Ok(self)
}
fn with_status(&self, status: BatchItemStatus) -> impl Iterator<Item = &BatchItem<O>> + '_ {
self.items.iter().filter(move |item| item.status == status)
}
}
impl BatchResult<crate::future::BranchValue> {
pub fn value<T: 'static>(&self, key: &crate::BranchKey<T>) -> Option<&T> {
self.items
.iter()
.find(|item| item.index == key.index())?
.result
.as_ref()?
.of_branch(key)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum NestingMode {
#[default]
Normal,
Flat,
}
pub(crate) type ItemNamer<I> = Box<dyn Fn(&I, usize) -> String + Send + Sync>;
pub(crate) type BatchSummary<O> = Arc<dyn Fn(&BatchResult<O>) -> String + Send + Sync>;
pub(crate) struct MapExecution<I, O, F, IS, RS> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) max_concurrency: Option<usize>,
pub(crate) completion: Option<crate::builders::map_parallel::CompletionConfig>,
pub(crate) serdes: IS,
pub(crate) encoding: ItemEncoding,
pub(crate) result_serdes: RS,
pub(crate) nesting: NestingMode,
pub(crate) item_namer: Option<ItemNamer<I>>,
pub(crate) summary: Option<BatchSummary<O>>,
pub(crate) items: Vec<I>,
pub(crate) closure: Arc<F>,
pub(crate) _marker: std::marker::PhantomData<fn() -> O>,
}
impl<I, O, F, Fut, IS, RS> MapExecution<I, O, F, IS, RS>
where
I: Send + 'static,
O: Send + 'static,
F: Fn(DurableContext, I, usize) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
IS: Serdes<O>,
RS: Serdes<String>,
{
pub(crate) fn execute_batch_result_started(
self,
) -> impl Future<Output = Result<BatchResult<O>, OperationError>> + Send + 'static {
let started = self.start();
self.run(started)
}
fn start(&self) -> Result<StartStep, OperationError> {
batch_start(
&self.ctx,
&self.op_id,
self.name.as_deref(),
MAP_SUB_TYPE,
self.max_concurrency,
)
}
async fn run(
self,
started: Result<StartStep, OperationError>,
) -> Result<BatchResult<O>, OperationError> {
let started = started?;
let total_items = self.items.len();
let user_names: Vec<Option<String>> = self
.items
.iter()
.enumerate()
.map(|(i, item)| self.item_namer.as_ref().map(|namer| namer(item, i)))
.collect();
drop(self.item_namer);
let record_names = record_names_of(&user_names, "map-item");
let items = into_item_slots(self.items);
let closure = self.closure;
let items_ref = Arc::clone(&items);
let item_ids = ItemIds::new(&self.op_id, total_items);
let dispatch = MapDispatch {
run_item: Arc::new(move |child_ctx: DurableContext, index: usize| {
let items = Arc::clone(&items_ref);
let closure = Arc::clone(&closure);
async move {
let item = take_item(&items, index).map_err(ChildFnError::from)?;
(closure)(child_ctx, item, index).await.map_err(Into::into)
}
}),
};
execute_batch(
self.ctx,
self.op_id,
self.name,
self.max_concurrency,
self.completion,
self.serdes,
self.encoding,
self.result_serdes,
self.nesting,
user_names,
record_names,
self.summary,
total_items,
MAP_SUB_TYPE,
MAP_ITERATION_SUB_TYPE,
&dispatch,
&item_ids,
started,
)
.await
}
}
pub(crate) struct ParallelExecution<O, IS, RS> {
pub(crate) ctx: DurableContext,
pub(crate) op_id: OperationId,
pub(crate) name: Option<String>,
pub(crate) max_concurrency: Option<usize>,
pub(crate) completion: Option<crate::builders::map_parallel::CompletionConfig>,
pub(crate) serdes: IS,
pub(crate) encoding: ItemEncoding,
pub(crate) result_serdes: RS,
pub(crate) nesting: NestingMode,
pub(crate) branches: Vec<(Option<String>, crate::future::BranchBody<O>)>,
pub(crate) summary: Option<BatchSummary<O>>,
pub(crate) item_ids: ItemIds,
}
impl<O, IS, RS> ParallelExecution<O, IS, RS>
where
O: Send + 'static,
IS: Serdes<O>,
RS: Serdes<String>,
{
pub(crate) fn execute_batch_result_started(
self,
) -> impl Future<Output = Result<BatchResult<O>, OperationError>> + Send + 'static {
let started = self.start();
self.run(started)
}
fn start(&self) -> Result<StartStep, OperationError> {
batch_start(
&self.ctx,
&self.op_id,
self.name.as_deref(),
PARALLEL_SUB_TYPE,
self.max_concurrency,
)
}
async fn run(
self,
started: Result<StartStep, OperationError>,
) -> Result<BatchResult<O>, OperationError> {
let started = started?;
let total = self.branches.len();
let mut user_names: Vec<Option<String>> = Vec::with_capacity(total);
let mut slots: Vec<std::sync::Mutex<Option<crate::future::BranchBody<O>>>> =
Vec::with_capacity(total);
for (name, body) in self.branches {
user_names.push(name);
slots.push(std::sync::Mutex::new(Some(body)));
}
let record_names = record_names_of(&user_names, "parallel-branch");
let dispatch = BranchDispatch {
slots: Arc::new(slots),
};
execute_batch(
self.ctx,
self.op_id,
self.name,
self.max_concurrency,
self.completion,
self.serdes,
self.encoding,
self.result_serdes,
self.nesting,
user_names,
record_names,
self.summary,
total,
PARALLEL_SUB_TYPE,
PARALLEL_BRANCH_SUB_TYPE,
&dispatch,
&self.item_ids,
started,
)
.await
}
}
fn record_names_of(user_names: &[Option<String>], prefix: &str) -> Vec<String> {
user_names
.iter()
.enumerate()
.map(|(i, name)| match name.as_deref() {
Some(name) if !name.is_empty() => name.to_owned(),
_ => format!("{prefix}-{i}"),
})
.collect()
}
pub(crate) fn collect_successful<O>(
batch_result: BatchResult<O>,
) -> Result<Vec<O>, OperationError> {
if let Some(err) = await_error(&batch_result) {
return Err(err);
}
Ok(batch_result
.items
.into_iter()
.filter(|item| item.status == BatchItemStatus::Succeeded)
.filter_map(|item| item.result)
.collect())
}
fn await_error<O>(batch: &BatchResult<O>) -> Option<OperationError> {
if !matches!(
batch.reason,
CompletionReason::FailureToleranceExceeded | CompletionReason::CustomCompletionFailed
) {
return None;
}
if let Some(item) = batch
.items
.iter()
.find(|item| item.status == BatchItemStatus::Failed)
{
return Some(item_failure_error(item.failure_wire()));
}
Some(match batch.reason {
CompletionReason::FailureToleranceExceeded => batch_error("failure tolerance exceeded"),
_ => batch_completion_failed_error(),
})
}
#[derive(Clone)]
pub(crate) struct ItemIds(Arc<[OperationId]>);
impl ItemIds {
pub(crate) fn new(batch: &OperationId, count: usize) -> Self {
Self(
(1..=count)
.map(|n| OperationId::new(format!("{}-{n}", batch.positional())))
.collect(),
)
}
#[expect(clippy::expect_used)]
fn id(&self, index: usize) -> &OperationId {
self.0
.get(index)
.expect("every caller passes an index below the item count")
}
}
pub(crate) struct MixedItemSerdes {
by_op_id: std::collections::HashMap<String, usize>,
decoders: Arc<Vec<Arc<dyn crate::future::BranchDecoder>>>,
}
impl MixedItemSerdes {
pub(crate) fn new(
item_ids: &ItemIds,
decoders: Arc<Vec<Arc<dyn crate::future::BranchDecoder>>>,
) -> Self {
let mut by_op_id = std::collections::HashMap::new();
for index in 0..decoders.len() {
by_op_id.insert(item_ids.id(index).wire().to_owned(), index);
}
Self { by_op_id, decoders }
}
}
impl Serdes<crate::future::BranchValue> for MixedItemSerdes {
async fn serialize(
&self,
value: crate::future::BranchValue,
context: SerdesContext,
) -> Result<String, BoxError> {
value.encode(context).await
}
async fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> Result<crate::future::BranchValue, BoxError> {
let decoder = self
.by_op_id
.get(context.operation_id())
.and_then(|index| self.decoders.get(*index))
.ok_or_else(|| {
format!(
"parallel_mixed: no branch has operation ID \"{}\"",
context.operation_id()
)
})?;
decoder.decode(wire, context).await
}
}
pub(crate) fn publish_outcome(
result: Result<BatchResult<crate::future::BranchValue>, OperationError>,
decoders: &[Arc<dyn crate::future::BranchDecoder>],
) -> Result<CompletionReason, OperationError> {
let batch = result?;
let error = await_error(&batch);
let reason = batch.reason;
for item in batch.items {
match item.status {
BatchItemStatus::Succeeded => {
if let Some(value) = item.result {
value.publish();
}
}
BatchItemStatus::Failed => {
if let Some(decoder) = decoders.get(item.index) {
decoder.publish_error(item_failure_error(item.failure_wire()));
}
}
BatchItemStatus::Started => {}
}
}
error.map_or(Ok(reason), Err)
}
struct PreClaimed {
index: usize,
op_id: OperationId,
is_terminal: bool,
has_record: bool,
}
type ItemJoin<O> = (usize, Result<ItemOutcome<O>, OperationError>);
type ItemOutcome<O> = (BatchItem<O>, Option<String>);
struct ItemRequest<IS> {
ctx: DurableContext,
child_op_id: OperationId,
index: usize,
is_terminal: bool,
parent_wire: String,
child_sub_type: String,
item_name: String,
nesting: NestingMode,
start_info: Arc<crate::plugin::OperationInfo>,
serdes: Arc<IS>,
start: std::sync::Mutex<Option<SubmittedWrite>>,
}
impl<IS> ItemRequest<IS> {
async fn await_start(&self, action: OperationAction) -> Result<(), OperationError> {
let start = self
.start
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
self.ctx
.await_retained_start(self.child_op_id.wire(), start, action)
.await
}
}
enum ItemPrelude<O> {
Done(ItemOutcome<O>),
Run {
child_ctx: DurableContext,
},
}
async fn item_before<O, IS>(req: &ItemRequest<IS>) -> Result<ItemPrelude<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
let child_positional = req.child_op_id.positional().to_owned();
let child_wire = req.child_op_id.wire().to_owned();
let serdes_ctx = SerdesContext::new(child_wire.clone(), req.ctx.execution_arn());
if req.is_terminal {
match replay_terminal_child(
&req.ctx,
&child_positional,
req.index,
&req.item_name,
&req.serdes,
&serdes_ctx,
)
.await
{
Ok(item) => return Ok(ItemPrelude::Done(item)),
Err(e) => {
let is_replay_children =
crate::error::chain_string(&e).contains(REPLAY_CHILDREN_SENTINEL);
if !is_replay_children {
return Err(e);
}
}
}
}
if req.nesting == NestingMode::Flat {
let child_ctx =
req.ctx
.new_flat_child(&child_positional, &req.parent_wire, Some(&req.item_name));
return Ok(ItemPrelude::Run { child_ctx });
}
Ok(ItemPrelude::Run {
child_ctx: req.ctx.new_child_rebuilding(
&child_positional,
Some(&req.item_name),
req.is_terminal,
),
})
}
#[expect(clippy::too_many_lines)] async fn item_after<O, IS>(
req: &ItemRequest<IS>,
outcome: ItemBody<O>,
replaying: bool,
) -> Result<ItemOutcome<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
let outcome = match outcome {
ItemBody::Ran(ran) => ran.into_parts(),
ItemBody::Unhooked(result) => result.map_err(|error| {
let wire = error.failure_wire();
(error, wire)
}),
};
let child_wire = req.child_op_id.wire().to_owned();
let serdes_ctx = SerdesContext::new(child_wire.clone(), req.ctx.execution_arn());
let serdes = &req.serdes;
let writer = &req.ctx;
if req.nesting == NestingMode::Flat || req.is_terminal {
let settled = match outcome {
Ok(value) => {
let round_trip = async {
let name = item_fault_name(&req.item_name);
let serialized =
serialize_value(&req.ctx, name, value, serdes, serdes_ctx.clone()).await?;
let deserialized: O =
deserialize_value(&req.ctx, name, serialized.clone(), serdes, serdes_ctx)
.await?;
Ok::<_, OperationError>((deserialized, serialized))
};
match round_trip.await {
Ok((deserialized, serialized)) => Ok((
BatchItem {
index: req.index,
name: req.item_name.clone(),
status: BatchItemStatus::Succeeded,
result: Some(deserialized),
error_message: None,
error_type: None,
error_data: None,
stack_trace: Vec::new(),
},
Some(serialized),
)),
Err(op_err) if req.is_terminal => Err(op_err),
Err(op_err) => {
let wire = crate::error::serialization_failure_wire(&op_err);
Ok((
BatchItem::failed_from_wire(req.index, req.item_name.clone(), &wire),
None,
))
}
}
}
Err((_error, wire)) => {
Ok((
BatchItem::failed_from_wire(req.index, req.item_name.clone(), &wire),
None,
))
}
};
if req.nesting == NestingMode::Flat
&& let Ok((item, _)) = &settled
{
if matches!(item.status, BatchItemStatus::Succeeded) {
item_ended(
req,
crate::OperationStatus::Succeeded,
None,
None,
replaying,
);
} else {
let wire = item.failure_wire();
item_ended(
req,
crate::OperationStatus::Failed,
None,
Some(&wire),
replaying,
);
}
}
return settled;
}
match outcome {
Ok(value) => {
let serialized = match serialize_value(
&req.ctx,
item_fault_name(&req.item_name),
value,
serdes,
serdes_ctx.clone(),
)
.await
{
Ok(serialized) => serialized,
Err(op_err) => {
req.await_start(OperationAction::Fail).await?;
let wire = crate::error::serialization_failure_wire(&op_err);
let update = build_child_fail_update(
&child_wire,
&req.item_name,
&req.child_sub_type,
&req.parent_wire,
&wire,
req.ctx.preserves_children(1),
);
if let Err(client_err) = writer.checkpoint_updates(vec![update]).await {
return writer.checkpoint_failure_unrecoverable(client_err).await;
}
item_ended(
req,
crate::OperationStatus::Failed,
None,
Some(&wire),
replaying,
);
return Ok((
BatchItem::failed_from_wire(req.index, req.item_name.clone(), &wire),
None,
));
}
};
let mut builder = OperationUpdate::builder()
.id(child_wire.clone())
.r#type(OperationType::Context)
.sub_type(req.child_sub_type.clone())
.action(OperationAction::Succeed)
.parent_id(req.parent_wire.clone())
.name(req.item_name.clone());
if serialized.len() > CHECKPOINT_SIZE_LIMIT_BYTES {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
} else {
builder = builder.payload(serialized.clone());
if req.ctx.preserves_children(1) {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
let result = update.payload().map(str::to_owned);
req.await_start(OperationAction::Succeed).await?;
if let Err(err) = writer.checkpoint_updates(vec![update]).await {
return writer.checkpoint_failure_unrecoverable(err).await;
}
item_ended(
req,
crate::OperationStatus::Succeeded,
result,
None,
replaying,
);
let deserialized: O = deserialize_value(
&req.ctx,
item_fault_name(&req.item_name),
serialized.clone(),
serdes,
serdes_ctx,
)
.await?;
Ok((
BatchItem {
index: req.index,
name: req.item_name.clone(),
status: BatchItemStatus::Succeeded,
result: Some(deserialized),
error_message: None,
error_type: None,
error_data: None,
stack_trace: Vec::new(),
},
Some(serialized),
))
}
Err((_error, wire)) => {
let mut builder = OperationUpdate::builder()
.id(child_wire.clone())
.r#type(OperationType::Context)
.sub_type(req.child_sub_type.clone())
.action(OperationAction::Fail)
.parent_id(req.parent_wire.clone())
.name(req.item_name.clone())
.error(wire.to_error_object());
if req.ctx.preserves_children(1) {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
req.await_start(OperationAction::Fail).await?;
if let Err(client_err) = writer.checkpoint_updates(vec![update]).await {
return writer.checkpoint_failure_unrecoverable(client_err).await;
}
item_ended(
req,
crate::OperationStatus::Failed,
None,
Some(&wire),
replaying,
);
Ok((
BatchItem::failed_from_wire(req.index, req.item_name.clone(), &wire),
None,
))
}
}
}
enum ItemBody<O> {
Ran(crate::plugin::Ran<O, ChildFnError>),
Unhooked(Result<O, ChildFnError>),
}
async fn run_single_item<O, F, Fut, IS>(
req: ItemRequest<IS>,
run_item: Arc<F>,
) -> Result<ItemOutcome<O>, OperationError>
where
O: Send + 'static,
F: Fn(DurableContext, usize) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, ChildFnError>> + Send + 'static,
IS: Serdes<O>,
{
match item_before(&req).await? {
ItemPrelude::Done(item) => Ok(item),
ItemPrelude::Run { child_ctx } => {
let span = child_ctx.replay_span();
let index = req.index;
let body = UserBody::Context(child_ctx.prefix().to_owned());
let item_ctx = child_ctx.clone();
let scope = item_scope(&req);
let outcome = if req.is_terminal {
let outcome =
run_body(body.into(), async move { run_item(child_ctx, index).await })
.instrument(span)
.await;
drop(scope);
ItemBody::Unhooked(outcome)
} else {
let info = user_function_info(&req, &item_ctx);
ItemBody::Ran(
crate::plugin::run_user_function(
item_ctx.shared_plugins(),
info,
body,
span,
move || async move {
let outcome = run_item(child_ctx, index).await;
drop(scope);
outcome
},
ChildFnError::failure_wire,
)
.await,
)
};
item_after(&req, outcome, item_ctx.is_replaying()).await
}
}
}
fn user_function_info<IS>(
req: &ItemRequest<IS>,
item_ctx: &DurableContext,
) -> Arc<crate::plugin::OperationInfo> {
let is_replay = item_ctx.is_replaying();
if req.start_info.is_replay() == is_replay {
Arc::clone(&req.start_info)
} else {
Arc::new((*req.start_info).clone().with_replay(is_replay))
}
}
async fn run_branch_item<O, IS>(
req: ItemRequest<IS>,
slots: Arc<Vec<std::sync::Mutex<Option<crate::future::BranchBody<O>>>>>,
) -> Result<ItemOutcome<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
match item_before(&req).await? {
ItemPrelude::Done(item) => Ok(item),
ItemPrelude::Run { child_ctx } => {
let span = child_ctx.replay_span();
let item_ctx = child_ctx.clone();
let scope = item_scope(&req);
let outcome = match take_branch_body(&slots, req.index) {
Ok(body) => {
let active = UserBody::Context(child_ctx.prefix().to_owned());
if req.is_terminal {
let outcome = run_body(active.into(), body.start(child_ctx))
.instrument(span)
.await;
drop(scope);
ItemBody::Unhooked(outcome)
} else {
let info = user_function_info(&req, &item_ctx);
ItemBody::Ran(
crate::plugin::run_user_function(
item_ctx.shared_plugins(),
info,
active,
span,
move || async move {
let outcome = body.start(child_ctx).await;
drop(scope);
outcome
},
ChildFnError::failure_wire,
)
.await,
)
}
}
Err(err) => {
drop(scope);
ItemBody::Unhooked(Err(err))
}
};
item_after(&req, outcome, item_ctx.is_replaying()).await
}
}
}
fn item_scope<IS>(req: &ItemRequest<IS>) -> ScopeClose {
ScopeClose::new(req.ctx.admission(), req.child_op_id.wire())
}
fn take_branch_body<O>(
slots: &[std::sync::Mutex<Option<crate::future::BranchBody<O>>>],
index: usize,
) -> Result<crate::future::BranchBody<O>, ChildFnError> {
let guard = slots
.get(index)
.ok_or_else(|| ChildFnError::new("branch index out of bounds"))?;
let mut lock = guard
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
lock.take()
.ok_or_else(|| ChildFnError::new("branch already consumed (concurrent access bug)"))
}
trait ItemDispatch<O, IS>: Send + Sync {
fn spawn_item(
&self,
set: &mut TrackedJoinSet<ItemJoin<O>>,
req: ItemRequest<IS>,
) -> Result<Option<tokio::task::Id>, NoRuntime>;
}
struct MapDispatch<F> {
run_item: Arc<F>,
}
impl<O, F, Fut, IS> ItemDispatch<O, IS> for MapDispatch<F>
where
O: Send + 'static,
F: Fn(DurableContext, usize) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, ChildFnError>> + Send + 'static,
IS: Serdes<O>,
{
fn spawn_item(
&self,
set: &mut TrackedJoinSet<ItemJoin<O>>,
req: ItemRequest<IS>,
) -> Result<Option<tokio::task::Id>, NoRuntime> {
let run_item = Arc::clone(&self.run_item);
let ownership = Arc::clone(req.ctx.task_ownership());
set.spawn(
crate::task::TaskContext::current(),
&ownership,
async move {
let index = req.index;
(index, run_single_item(req, run_item).await)
},
)
}
}
struct BranchDispatch<O> {
slots: Arc<Vec<std::sync::Mutex<Option<crate::future::BranchBody<O>>>>>,
}
impl<O, IS> ItemDispatch<O, IS> for BranchDispatch<O>
where
O: Send + 'static,
IS: Serdes<O>,
{
fn spawn_item(
&self,
set: &mut TrackedJoinSet<ItemJoin<O>>,
req: ItemRequest<IS>,
) -> Result<Option<tokio::task::Id>, NoRuntime> {
let slots = Arc::clone(&self.slots);
let ownership = Arc::clone(req.ctx.task_ownership());
set.spawn(
crate::task::TaskContext::current(),
&ownership,
async move {
let index = req.index;
(index, run_branch_item(req, slots).await)
},
)
}
}
struct BatchProgress<O> {
tracker: CompletionTracker,
results: Vec<Option<BatchItem<O>>>,
wires: Vec<Option<String>>,
started: Vec<bool>,
}
impl<O> BatchProgress<O> {
fn new(names: Vec<Option<String>>) -> Self {
let total_items = names.len();
Self {
tracker: CompletionTracker::new(names),
results: (0..total_items).map(|_| None).collect(),
wires: vec![None; total_items],
started: vec![false; total_items],
}
}
fn mark_started(&mut self, index: usize) {
if let Some(slot) = self.started.get_mut(index) {
*slot = true;
}
}
fn mark_dispatched(&mut self, index: usize) {
self.mark_started(index);
self.tracker.mark_dispatched(index);
}
fn settle_outcome(
&mut self,
completion_cfg: &crate::builders::map_parallel::CompletionConfig,
total_items: usize,
index: usize,
(item, wire): ItemOutcome<O>,
) {
self.tracker
.settle(completion_cfg, total_items, index, item.status);
if let Some(slot) = self.results.get_mut(index) {
*slot = Some(item);
}
if let Some(slot) = self.wires.get_mut(index) {
*slot = wire;
}
}
}
struct BatchEnv<'a, O, IS> {
ctx: &'a DurableContext,
dispatch: &'a dyn ItemDispatch<O, IS>,
completion_cfg: &'a crate::builders::map_parallel::CompletionConfig,
total_items: usize,
nesting: NestingMode,
parent_wire: &'a str,
child_sub_type: &'a str,
serdes: &'a Arc<IS>,
}
impl<O, IS> BatchEnv<'_, O, IS>
where
O: Send + 'static,
IS: Serdes<O>,
{
fn settle_joined(
&self,
joined: TaskOutcome<ItemJoin<O>>,
progress: &mut BatchProgress<O>,
) -> Result<(), OperationError> {
let (index, outcome) = joined.into_value();
let item = outcome?;
progress.settle_outcome(self.completion_cfg, self.total_items, index, item);
Ok(())
}
async fn resolve_terminal_inline(
&self,
op_id: &OperationId,
index: usize,
item_name: &str,
join_set: &mut TrackedJoinSet<ItemJoin<O>>,
progress: &mut BatchProgress<O>,
) -> Result<(), OperationError> {
let child_positional = op_id.positional().to_owned();
let serdes_ctx = SerdesContext::new(op_id.wire().to_owned(), self.ctx.execution_arn());
match replay_terminal_child(
self.ctx,
&child_positional,
index,
item_name,
self.serdes,
&serdes_ctx,
)
.await
{
Ok(item) => {
progress.settle_outcome(self.completion_cfg, self.total_items, index, item);
Ok(())
}
Err(e) => {
let is_replay_children =
crate::error::chain_string(&e).contains(REPLAY_CHILDREN_SENTINEL);
if !is_replay_children {
return Err(e);
}
let req = ItemRequest {
ctx: self.ctx.clone(),
child_op_id: op_id.clone(),
index,
is_terminal: true,
parent_wire: self.parent_wire.to_owned(),
child_sub_type: self.child_sub_type.to_owned(),
item_name: item_name.to_owned(),
nesting: self.nesting,
start_info: Arc::new(item_info(
self.ctx,
op_id.wire(),
self.child_sub_type,
item_name,
)),
serdes: Arc::clone(self.serdes),
start: std::sync::Mutex::new(None),
};
let reconstruction_id = self
.dispatch
.spawn_item(join_set, req)
.map_err(NoRuntime::operation_error)?;
let Some((joined_id, joined)) = join_set.join_next_with_id().await else {
return Err(batch_error(
"re-executed replay child produced no join event",
));
};
if Some(joined_id) != reconstruction_id {
return Err(batch_error(
"re-executed replay child joined an unexpected task",
));
}
self.settle_joined(joined, progress)
}
}
}
}
#[expect(clippy::too_many_lines)]
#[expect(clippy::too_many_arguments)] async fn execute_batch<O, IS, RS>(
ctx: DurableContext,
parent_op_id: OperationId,
parent_name: Option<String>,
max_concurrency: Option<usize>,
completion: Option<crate::builders::map_parallel::CompletionConfig>,
serdes: IS,
encoding: ItemEncoding,
result_serdes: RS,
nesting: NestingMode,
user_names: Vec<Option<String>>,
record_names: Vec<String>,
summary: Option<BatchSummary<O>>,
total_items: usize,
parent_sub_type: &str,
child_sub_type: &str,
dispatch: &dyn ItemDispatch<O, IS>,
item_ids: &ItemIds,
started: StartStep,
) -> Result<BatchResult<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
RS: Serdes<String>,
{
let serdes = Arc::new(serdes);
let (recorded, parent_start) = match started {
StartStep::Refused => return std::future::pending().await,
StartStep::Recorded(view) => (Some(view), None),
StartStep::Queued(write) => (None, Some(write)),
};
let resumed = recorded.is_some();
let rebuilding = recorded
.as_ref()
.is_some_and(|view| view.status.is_terminal());
#[expect(clippy::expect_used)]
let item_name_of = |i: usize| -> String {
record_names
.get(i)
.expect("record_names has one entry per input item")
.clone()
};
let parent_positional = parent_op_id.positional().to_owned();
let parent_wire = parent_op_id.wire().to_owned();
let batch_ctx =
ctx.new_child_rebuilding(&parent_positional, parent_name.as_deref(), rebuilding);
let mut replay_plan: Option<ReplayPlan> = None;
if let Some(view) = recorded
&& view.status.is_terminal()
{
let snapshot = ctx
.checkpoint_terminal_replay(&parent_positional)
.ok_or_else(|| batch_error("terminal batch has no checkpoint record"))?;
let serdes_ctx = SerdesContext::new(&parent_wire, ctx.execution_arn());
let decision =
if matches!(snapshot.status, CheckpointStatus::Succeeded) && snapshot.replay_children {
match parse_replay_children_payload(snapshot.result.as_deref())? {
Some(record) => Some(record),
None => Some(
full_result_decision(
&ctx,
parent_name.as_deref(),
snapshot.result.clone().unwrap_or_default(),
&result_serdes,
&serdes_ctx,
)
.await?,
),
}
} else {
None
};
if let Some(record) = decision {
replay_plan = Some(ReplayPlan::from_record(&record, total_items)?);
} else {
match replay_terminal_batch(
&ctx,
parent_name.as_deref(),
&snapshot,
&serdes,
encoding.clone(),
total_items,
&result_serdes,
&serdes_ctx,
item_ids,
)
.await
{
Ok(result) => {
ctx.emit_operation_replayed(
&parent_wire,
parent_name.as_deref(),
"Context",
Some(parent_sub_type),
view.attempt,
);
return Ok(result);
}
Err(e) => {
let is_replay_children =
crate::error::chain_string(&e).contains(REPLAY_CHILDREN_SENTINEL);
if !is_replay_children {
if matches!(snapshot.status, CheckpointStatus::Failed) {
ctx.emit_operation_replayed(
&parent_wire,
parent_name.as_deref(),
"Context",
Some(parent_sub_type),
view.attempt,
);
}
return Err(e);
}
}
}
}
}
let completion_cfg = completion.unwrap_or_default();
let rebuild = replay_plan.is_some();
let active = UserBody::Context(batch_ctx.prefix().to_owned());
let span = batch_ctx.replay_span();
let produce = || {
produce_batch(BatchSection {
ctx: &ctx,
batch_ctx: &batch_ctx,
max_concurrency,
completion_cfg: &completion_cfg,
serdes: &serdes,
nesting,
user_names,
item_name_of: &item_name_of,
total_items,
child_sub_type,
dispatch,
item_ids,
parent_wire: &parent_wire,
replay_plan,
})
};
let produced = if rebuild {
run_body(active.into(), produce()).instrument(span).await
} else {
let info = container_info(&ctx, &parent_wire, parent_sub_type, parent_name.as_deref());
let info = if resumed {
info
} else {
info.with_status(crate::OperationStatus::Started)
};
let info = Arc::new(info.with_replay(batch_ctx.is_replaying()));
match crate::plugin::run_user_function(
ctx.shared_plugins(),
info,
active,
span,
move || async move { Ok::<_, std::convert::Infallible>(produce().await) },
crate::plugin::never_fails,
)
.await
.into_parts()
{
Ok(produced) => produced,
Err((never, _)) => match never {},
}
};
let Produced {
result: batch_result,
wires: final_wires,
} = produced?;
if rebuild {
return Ok(batch_result);
}
let payload = from_batch_result(&batch_result, &final_wires, &encoding);
let serialized_payload = serialize_value(
&ctx,
parent_name.as_deref(),
payload,
&result_serdes,
SerdesContext::new(&parent_wire, ctx.execution_arn()),
)
.await
.map_err(|err| {
let wire = crate::error::serialization_failure_wire(&err);
err.with_wire(wire)
})?;
let caller_summary = match summary {
Some(summary) if serialized_payload.len() > CHECKPOINT_SIZE_LIMIT_BYTES => {
Some(summary(&batch_result))
}
_ => None,
};
let summary_json =
oversized_batch_summary(&batch_result, parent_sub_type, caller_summary.as_deref());
checkpoint_batch_success_serialized(
&ctx,
&parent_wire,
parent_name.as_deref(),
parent_sub_type,
&serialized_payload,
&summary_json,
parent_start,
)
.await?;
Ok(batch_result)
}
struct Produced<O> {
result: BatchResult<O>,
wires: Vec<Option<String>>,
}
struct BatchSection<'a, O, IS> {
ctx: &'a DurableContext,
batch_ctx: &'a DurableContext,
max_concurrency: Option<usize>,
completion_cfg: &'a crate::builders::map_parallel::CompletionConfig,
serdes: &'a Arc<IS>,
nesting: NestingMode,
user_names: Vec<Option<String>>,
item_name_of: &'a (dyn Fn(usize) -> String + Send + Sync),
total_items: usize,
child_sub_type: &'a str,
dispatch: &'a dyn ItemDispatch<O, IS>,
item_ids: &'a ItemIds,
parent_wire: &'a str,
replay_plan: Option<ReplayPlan>,
}
#[expect(clippy::too_many_lines)]
async fn produce_batch<O, IS>(
section: BatchSection<'_, O, IS>,
) -> Result<Produced<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
let BatchSection {
ctx,
batch_ctx,
max_concurrency,
completion_cfg,
serdes,
nesting,
user_names,
item_name_of,
total_items,
child_sub_type,
dispatch,
item_ids,
parent_wire,
replay_plan,
} = section;
if total_items == 0 {
let reason = if let Some(plan) = &replay_plan {
plan.reason
} else {
let decision = completion_cfg.predicate_decision(&BatchStats::from_counts(0, 0, &[]));
custom_reason(decision).unwrap_or(CompletionReason::AllCompleted)
};
return Ok(Produced {
result: BatchResult {
items: Vec::new(),
reason,
},
wires: Vec::new(),
});
}
let concurrency = max_concurrency.unwrap_or(total_items).max(1);
let mut pre_claimed: Vec<PreClaimed> = Vec::with_capacity(total_items);
if concurrency > 1 {
for i in 0..total_items {
let child_op_id = item_ids.id(i).clone();
let consult_checkpoint = replay_plan.as_ref().is_none_or(|plan| plan.admits(i));
let (has_record, is_terminal) = if consult_checkpoint {
let child_positional = child_op_id.positional().to_owned();
let child_wire = child_op_id.wire().to_owned();
let view = ctx.checkpoint_view_validated(
&child_positional,
&child_wire,
"Context",
Some(child_sub_type),
Some(&item_name_of(i)),
)?;
(view.is_some(), view.is_some_and(|v| v.status.is_terminal()))
} else {
(false, false)
};
pre_claimed.push(PreClaimed {
index: i,
op_id: child_op_id,
is_terminal,
has_record,
});
}
}
let child_sub_type_owned = child_sub_type.to_owned();
let mut join_set: TrackedJoinSet<ItemJoin<O>> =
TrackedJoinSet::new(Arc::clone(ctx.coordinator()));
let mut progress = BatchProgress::<O>::new(user_names);
if replay_plan.is_some() {
progress.tracker.stop_evaluating();
}
let mut running: usize = 0;
let mut batch_scope = Some(ScopeClose::new(ctx.admission(), parent_wire));
let mut next_index: usize = 0;
for pre in &pre_claimed {
if pre.has_record {
progress.mark_started(pre.index);
}
}
let env = BatchEnv {
ctx: batch_ctx,
dispatch,
completion_cfg,
total_items,
nesting,
parent_wire,
child_sub_type: &child_sub_type_owned,
serdes,
};
if concurrency > 1 {
progress.tracker.replaying_recorded = true;
for i in 0..total_items {
let Some(pre) = pre_claimed.get(i) else {
return Err(batch_error("pre-claimed index out of range"));
};
if !pre.is_terminal {
continue;
}
if let Some(plan) = &replay_plan
&& !plan.admits(i)
{
continue;
}
let item_name = item_name_of(i);
env.resolve_terminal_inline(&pre.op_id, i, &item_name, &mut join_set, &mut progress)
.await?;
}
progress
.tracker
.finish_replay_pass(completion_cfg, total_items);
}
loop {
while (replay_plan.is_some() || progress.tracker.stop_reason.is_none())
&& next_index < total_items
&& running < concurrency
{
let i = next_index;
next_index += 1;
if let Some(plan) = &replay_plan
&& !plan.admits(i)
{
if i < plan.total_count {
let item_name = item_name_of(i);
if let Some(slot) = progress.results.get_mut(i) {
*slot = Some(BatchItem {
index: i,
name: item_name,
status: BatchItemStatus::Started,
result: None,
error_message: None,
error_type: None,
error_data: None,
stack_trace: Vec::new(),
});
}
}
continue;
}
if concurrency > 1 && pre_claimed.get(i).is_some_and(|pre| pre.is_terminal) {
continue;
}
let pc = if concurrency > 1 {
let pre = pre_claimed
.get(i)
.ok_or_else(|| batch_error("pre-claimed index out of range"))?;
PreClaimed {
index: pre.index,
op_id: pre.op_id.clone(),
is_terminal: pre.is_terminal,
has_record: pre.has_record,
}
} else {
let child_op_id = item_ids.id(i).clone();
let view = ctx.checkpoint_view_validated(
child_op_id.positional(),
child_op_id.wire(),
"Context",
Some(child_sub_type),
Some(&item_name_of(i)),
)?;
let has_record = view.is_some();
let is_terminal = view.is_some_and(|v| v.status.is_terminal());
PreClaimed {
index: i,
op_id: child_op_id,
is_terminal,
has_record,
}
};
if pc.is_terminal {
progress.mark_dispatched(i);
let item_name = item_name_of(pc.index);
env.resolve_terminal_inline(
&pc.op_id,
pc.index,
&item_name,
&mut join_set,
&mut progress,
)
.await?;
continue;
}
if replay_plan.is_none() && progress.tracker.dispatch_gate(completion_cfg) {
break;
}
progress.mark_dispatched(i);
let item_name = item_name_of(pc.index);
let branch_index = pc.index;
let branch_wire = pc.op_id.wire().to_owned();
let reserved = (nesting != NestingMode::Flat && !pc.has_record).then(|| {
ctx.reserve_checkpoint_updates(vec![build_child_update(
&branch_wire,
&item_name,
&child_sub_type_owned,
parent_wire,
OperationAction::Start,
)])
});
let info = item_info(batch_ctx, &branch_wire, &child_sub_type_owned, &item_name);
let info = if nesting == NestingMode::Flat {
info.with_virtual(true)
.with_replay(batch_ctx.namespace_is_replaying(pc.op_id.positional()))
} else if pc.has_record {
info.with_replay(true)
} else {
info.with_status(crate::OperationStatus::Started)
.with_replay(false)
};
if let Some(plugins) = ctx.plugins() {
crate::plugin::operation_started(plugins, &info);
}
let start = reserved.map(crate::context::ReservedWrite::release);
let req = ItemRequest {
ctx: batch_ctx.clone(),
child_op_id: pc.op_id,
index: branch_index,
is_terminal: pc.is_terminal,
parent_wire: parent_wire.to_owned(),
child_sub_type: child_sub_type_owned.clone(),
item_name: item_name.clone(),
nesting,
start_info: Arc::new(info),
serdes: Arc::clone(serdes),
start: std::sync::Mutex::new(start),
};
dispatch
.spawn_item(&mut join_set, req)
.map_err(NoRuntime::operation_error)?;
running += 1;
}
if running == 0 {
break;
}
let Some((_, joined)) = join_set.join_next_with_id().await else {
break; };
running -= 1;
env.settle_joined(joined, &mut progress)?;
if replay_plan.is_none() && progress.tracker.stop_reason.is_some() {
drop(batch_scope.take());
join_set.abort_all();
break;
}
}
drop(batch_scope);
drop(join_set);
let BatchProgress {
tracker,
mut results,
wires,
started,
} = progress;
if replay_plan.is_none() {
for (i, (slot, started)) in results.iter_mut().zip(&started).enumerate() {
if slot.is_none() && *started {
*slot = Some(BatchItem {
index: i,
name: item_name_of(i),
status: BatchItemStatus::Started,
result: None,
error_message: None,
error_type: None,
error_data: None,
stack_trace: Vec::new(),
});
}
}
}
let (final_items, final_wires): (Vec<BatchItem<O>>, Vec<Option<String>>) = results
.into_iter()
.zip(wires)
.filter_map(|(slot, wire)| slot.map(|item| (item, wire)))
.unzip();
if let Some(plan) = replay_plan {
return Ok(Produced {
result: BatchResult {
items: final_items,
reason: plan.reason,
},
wires: final_wires,
});
}
let reason = tracker
.stop_reason
.unwrap_or(CompletionReason::AllCompleted);
Ok(Produced {
result: BatchResult {
items: final_items,
reason,
},
wires: final_wires,
})
}
fn batch_start(
ctx: &DurableContext,
op_id: &OperationId,
name: Option<&str>,
sub_type: &'static str,
max_concurrency: Option<usize>,
) -> Result<StartStep, OperationError> {
if !ctx.admission().admits(ctx.owners()) {
return Ok(StartStep::Refused);
}
ctx.enforce_task_ownership()?;
if max_concurrency == Some(0) {
ctx.coordinator()
.record_fatal("Error".to_owned(), MAX_CONCURRENCY_ZERO_MESSAGE.to_owned());
return Ok(StartStep::Refused);
}
let wire_id = op_id.wire();
crate::start::start_step(
ctx,
&crate::start::StartSpec {
op_id,
operation_type: "Context",
sub_type,
name,
order: crate::start::StartOrder::QueueThenHook,
},
|view| {
if view.status.is_terminal() {
crate::start::Cycle::Record
} else {
crate::start::Cycle::Resume
}
},
|| build_parent_update(wire_id, name, sub_type, OperationAction::Start, ctx),
|plugins, is_replay| {
let info = container_info(ctx, wire_id, sub_type, name);
let info = if is_replay {
info.with_replay(true)
} else {
info.with_status(crate::OperationStatus::Started)
.with_replay(false)
};
crate::plugin::operation_started(plugins, &info);
},
)
}
#[expect(clippy::too_many_arguments)] async fn replay_terminal_batch<O, IS, RS>(
ctx: &DurableContext,
name: Option<&str>,
snapshot: &crate::engine::TerminalReplaySnapshot,
serdes: &Arc<IS>,
encoding: ItemEncoding,
total_items: usize,
result_serdes: &RS,
serdes_ctx: &SerdesContext,
item_ids: &ItemIds,
) -> Result<BatchResult<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
RS: Serdes<String>,
{
match &snapshot.status {
CheckpointStatus::Succeeded => {
if snapshot.replay_children {
return Err(batch_error(REPLAY_CHILDREN_SENTINEL));
}
let payload_str = snapshot
.result
.clone()
.ok_or_else(|| batch_error("terminal batch has no result payload"))?;
let record_text: String =
deserialize_value(ctx, name, payload_str, result_serdes, serdes_ctx.clone())
.await?;
to_batch_result(
ctx,
&record_text,
serdes,
encoding,
total_items,
serdes_ctx.durable_execution_arn(),
item_ids,
)
.await
}
CheckpointStatus::Failed => {
let msg = snapshot.error_message.as_deref().unwrap_or("batch failed");
Err(batch_error(msg))
}
_ => {
Err(batch_error("unexpected non-terminal status in replay"))
}
}
}
async fn replay_terminal_child<O, S>(
ctx: &DurableContext,
child_positional: &str,
index: usize,
item_name: &str,
serdes: &S,
serdes_ctx: &SerdesContext,
) -> Result<ItemOutcome<O>, OperationError>
where
O: Send + 'static,
S: Serdes<O>,
{
let record = ctx
.checkpoint_terminal_replay(child_positional)
.ok_or_else(|| batch_error("replay child has no checkpoint record"))?;
match &record.status {
CheckpointStatus::Succeeded => {
if record.replay_children {
return Err(batch_error(REPLAY_CHILDREN_SENTINEL));
}
let payload = record
.result
.clone()
.ok_or_else(|| batch_error("succeeded child has no result"))?;
let value: O = deserialize_value(
ctx,
item_fault_name(item_name),
payload.clone(),
serdes,
serdes_ctx.clone(),
)
.await?;
Ok((
BatchItem {
index,
name: item_name.to_owned(),
status: BatchItemStatus::Succeeded,
result: Some(value),
error_message: None,
error_type: None,
error_data: None,
stack_trace: Vec::new(),
},
Some(payload),
))
}
CheckpointStatus::Failed => {
let msg = record.error_message.as_deref().unwrap_or("child failed");
Ok((
BatchItem {
index,
name: item_name.to_owned(),
status: BatchItemStatus::Failed,
result: None,
error_message: Some(msg.to_owned()),
error_type: record.error_type.clone(),
error_data: record.error_data.clone(),
stack_trace: record.stack_trace.clone().unwrap_or_default(),
},
None,
))
}
_ => Err(batch_error(
"unexpected non-terminal child status in replay",
)),
}
}
#[derive(Debug, serde::Deserialize)]
struct BatchDecisionRecord {
#[serde(rename = "totalCount")]
total_count: usize,
#[serde(rename = "completionReason")]
completion_reason: String,
#[serde(rename = "startedIndexes", default)]
started_indexes: Option<Vec<usize>>,
#[serde(rename = "completedIndexes", default)]
completed_indexes: Option<Vec<usize>>,
}
fn parse_replay_children_payload(
payload: Option<&str>,
) -> Result<Option<BatchDecisionRecord>, OperationError> {
let payload = payload.unwrap_or_default().trim();
if payload.is_empty() {
return Err(batch_error(
"malformed batch decision record: the payload is empty",
));
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(payload) else {
return Ok(None);
};
if let Some(statuses) = value.get("itemStatuses") {
let statuses = statuses
.as_str()
.filter(|s| s.chars().all(|c| matches!(c, 'S' | 'F' | '-')))
.ok_or_else(|| {
batch_error(
"malformed batch decision record: itemStatuses must be a string of S, F and -",
)
})?;
let completed = statuses
.chars()
.enumerate()
.filter_map(|(i, c)| (c != '-').then_some(i))
.collect();
return Ok(Some(BatchDecisionRecord {
total_count: statuses.chars().count(),
completion_reason: value
.get("completionReason")
.and_then(serde_json::Value::as_str)
.unwrap_or_default()
.to_owned(),
started_indexes: None,
completed_indexes: Some(completed),
}));
}
if value.get("totalCount").is_some()
&& (value.get("startedIndexes").is_some() || value.get("completedIndexes").is_some())
{
return parse_decision_record(payload).map(Some);
}
Ok(None)
}
async fn full_result_decision<RS>(
ctx: &DurableContext,
name: Option<&str>,
payload: String,
result_serdes: &RS,
serdes_ctx: &SerdesContext,
) -> Result<BatchDecisionRecord, OperationError>
where
RS: Serdes<String>,
{
let record_text: String =
deserialize_value(ctx, name, payload, result_serdes, serdes_ctx.clone()).await?;
let record = parse_batch_record(&record_text).map_err(|message| batch_error(&message))?;
check_record_indexes(&record)?;
let mut completed = Vec::with_capacity(record.all.len());
for item in &record.all {
if parse_item_status(&item.status)? != BatchItemStatus::Started {
completed.push(item.index);
}
}
Ok(BatchDecisionRecord {
total_count: record.all.len(),
completion_reason: record.completion_reason,
started_indexes: None,
completed_indexes: Some(completed),
})
}
fn check_record_indexes(record: &JsBatchRecord) -> Result<(), OperationError> {
for (position, item) in record.all.iter().enumerate() {
if item.index != position {
return Err(batch_error(&format!(
"batch record item at position {position} has index {}",
item.index
)));
}
}
Ok(())
}
fn parse_decision_record(payload: &str) -> Result<BatchDecisionRecord, OperationError> {
serde_json::from_str(payload.trim())
.map_err(|e| batch_error(&format!("malformed batch decision record: {e}")))
}
#[derive(Debug)]
struct ReplayPlan {
total_count: usize,
completed: Vec<bool>,
reason: CompletionReason,
}
impl ReplayPlan {
fn from_record(
record: &BatchDecisionRecord,
total_items: usize,
) -> Result<Self, OperationError> {
if record.total_count > total_items {
return Err(batch_error(&format!(
"invalid batch decision record: totalCount {} exceeds the input length {}",
record.total_count, total_items
)));
}
let Some(reason) = CompletionReason::try_from_wire(&record.completion_reason) else {
return Err(batch_error(&format!(
"invalid batch decision record: unknown completionReason {:?}",
record.completion_reason
)));
};
let total_count = record.total_count;
let (indexes, indexes_are_completed) =
match (&record.completed_indexes, &record.started_indexes) {
(Some(_), Some(_)) => {
return Err(batch_error(
"invalid batch decision record: both startedIndexes and \
completedIndexes are present",
));
}
(None, None) => {
return Err(batch_error(
"invalid batch decision record: neither startedIndexes nor \
completedIndexes is present",
));
}
(Some(completed), None) => (completed, true),
(None, Some(started)) => (started, false),
};
let mut completed = vec![!indexes_are_completed; total_count];
for &i in indexes {
let Some(slot) = completed.get_mut(i) else {
return Err(batch_error(&format!(
"invalid batch decision record: index {i} is outside [0, {total_count})",
)));
};
*slot = indexes_are_completed;
}
Ok(Self {
total_count,
completed,
reason,
})
}
fn admits(&self, index: usize) -> bool {
index < self.total_count && self.completed.get(index).copied().unwrap_or(false)
}
}
async fn checkpoint_batch_success_serialized(
ctx: &DurableContext,
parent_wire: &str,
parent_name: Option<&str>,
parent_sub_type: &str,
serialized_payload: &str,
summary_json: &str,
start: Option<SubmittedWrite>,
) -> Result<(), OperationError> {
let mut builder = OperationUpdate::builder()
.id(parent_wire.to_owned())
.r#type(OperationType::Context)
.sub_type(parent_sub_type.to_owned())
.action(OperationAction::Succeed);
if let Some(n) = parent_name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire_id) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire_id);
}
if serialized_payload.len() > CHECKPOINT_SIZE_LIMIT_BYTES {
if summary_json.len() > CHECKPOINT_SIZE_LIMIT_BYTES {
return Err(batch_error(
"batch summary exceeds the checkpoint size limit; \
refusing to record the batch outcome without it",
));
}
builder = builder
.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
)
.payload(summary_json.to_owned());
} else {
builder = builder.payload(serialized_payload.to_owned());
if ctx.preserves_children(1) {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
}
#[expect(clippy::expect_used)] let update = builder
.build()
.expect("all required OperationUpdate fields set");
let result = update.payload().map(str::to_owned);
ctx.await_retained_start(parent_wire, start, OperationAction::Succeed)
.await?;
if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
return ctx.checkpoint_failure_unrecoverable(err).await;
}
if let Some(plugins) = ctx.plugins() {
let info = container_info(ctx, parent_wire, parent_sub_type, parent_name)
.with_status(crate::OperationStatus::Succeeded)
.with_result(result)
.with_replay(false);
crate::plugin::operation_ended(plugins, &info);
}
Ok(())
}
fn container_info(
ctx: &DurableContext,
wire_id: &str,
sub_type: &str,
name: Option<&str>,
) -> crate::plugin::OperationInfo {
crate::plugin::OperationInfo::for_operation(
ctx,
wire_id,
crate::OperationKind::Context,
sub_type,
name,
)
}
fn item_info(
batch_ctx: &DurableContext,
wire_id: &str,
sub_type: &str,
name: &str,
) -> crate::plugin::OperationInfo {
crate::plugin::OperationInfo::for_operation(
batch_ctx,
wire_id,
crate::OperationKind::Context,
sub_type,
Some(name),
)
}
fn item_ended(
req: &ItemRequest<impl Sized>,
status: crate::OperationStatus,
result: Option<String>,
error: Option<&crate::error::WireError>,
replaying: bool,
) {
if let Some(plugins) = req.ctx.plugins() {
let info = item_info(
&req.ctx,
req.child_op_id.wire(),
&req.child_sub_type,
&req.item_name,
)
.with_status(status)
.with_result(result)
.with_error(error.cloned());
let info = if req.nesting == NestingMode::Flat {
info.with_virtual(true).with_replay(replaying)
} else {
info.with_replay(false)
};
crate::plugin::operation_ended(plugins, &info);
}
}
fn build_parent_update(
wire_id: &str,
name: Option<&str>,
sub_type: &str,
action: OperationAction,
ctx: &DurableContext,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(wire_id.to_owned())
.r#type(OperationType::Context)
.sub_type(sub_type.to_owned())
.action(action);
if let Some(n) = name {
builder = builder.name(n.to_owned());
}
if let Some(parent_wire) = ctx.parent_wire_id() {
builder = builder.parent_id(parent_wire);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_child_update(
child_wire: &str,
child_name: &str,
child_sub_type: &str,
parent_wire: &str,
action: OperationAction,
) -> OperationUpdate {
let builder = OperationUpdate::builder()
.id(child_wire.to_owned())
.r#type(OperationType::Context)
.sub_type(child_sub_type.to_owned())
.action(action)
.parent_id(parent_wire.to_owned())
.name(child_name.to_owned());
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
fn build_child_fail_update(
child_wire: &str,
child_name: &str,
child_sub_type: &str,
parent_wire: &str,
wire: &crate::error::WireError,
preserve: bool,
) -> OperationUpdate {
let mut builder = OperationUpdate::builder()
.id(child_wire.to_owned())
.r#type(OperationType::Context)
.sub_type(child_sub_type.to_owned())
.action(OperationAction::Fail)
.parent_id(parent_wire.to_owned())
.error(wire.to_error_object())
.name(child_name.to_owned());
if preserve {
builder = builder.context_options(
aws_sdk_lambda::types::ContextOptions::builder()
.replay_children(true)
.build(),
);
}
#[expect(clippy::expect_used)] builder
.build()
.expect("all required OperationUpdate fields set")
}
struct CompletionTracker {
success_count: usize,
failure_count: usize,
items: Vec<CompletionItem>,
stop_reason: Option<CompletionReason>,
replaying_recorded: bool,
evaluating: bool,
}
impl CompletionTracker {
fn new(names: Vec<Option<String>>) -> Self {
Self {
success_count: 0,
failure_count: 0,
items: names
.into_iter()
.enumerate()
.map(|(index, name)| CompletionItem::new(index, name, None))
.collect(),
stop_reason: None,
replaying_recorded: false,
evaluating: true,
}
}
fn stop_evaluating(&mut self) {
self.evaluating = false;
}
fn stats(&self) -> BatchStats<'_> {
BatchStats::from_counts(self.success_count, self.failure_count, &self.items)
}
fn mark_dispatched(&mut self, index: usize) {
if let Some(item) = self.items.get_mut(index) {
item.status = Some(BatchItemStatus::Started);
}
}
fn dispatch_gate(&mut self, cfg: &crate::builders::map_parallel::CompletionConfig) -> bool {
if self.stop_reason.is_none() {
self.stop_reason = custom_reason(cfg.predicate_decision(&self.stats()));
}
self.stop_reason.is_some()
}
fn settle(
&mut self,
cfg: &crate::builders::map_parallel::CompletionConfig,
total_items: usize,
index: usize,
status: BatchItemStatus,
) {
match status {
BatchItemStatus::Succeeded => self.success_count += 1,
BatchItemStatus::Failed => self.failure_count += 1,
BatchItemStatus::Started => return,
}
if let Some(item) = self.items.get_mut(index) {
item.status = Some(status);
}
if !self.evaluating {
return;
}
if self.stop_reason.is_none() && !self.replaying_recorded {
self.stop_reason =
evaluate_thresholds(cfg, self.success_count, self.failure_count, total_items);
}
if self.stop_reason.is_none() {
self.stop_reason = custom_reason(cfg.predicate_decision(&self.stats()));
}
}
fn finish_replay_pass(
&mut self,
cfg: &crate::builders::map_parallel::CompletionConfig,
total_items: usize,
) {
self.replaying_recorded = false;
if self.evaluating
&& self.stop_reason.is_none()
&& self.success_count + self.failure_count > 0
{
self.stop_reason =
evaluate_thresholds(cfg, self.success_count, self.failure_count, total_items);
}
}
}
fn custom_reason(decision: CompletionDecision) -> Option<CompletionReason> {
match decision {
CompletionDecision::Continue => None,
CompletionDecision::Complete(CompletionOutcome::Succeeded) => {
Some(CompletionReason::CustomCompletionSucceeded)
}
CompletionDecision::Complete(CompletionOutcome::Failed) => {
Some(CompletionReason::CustomCompletionFailed)
}
}
}
fn evaluate_thresholds(
cfg: &crate::builders::map_parallel::CompletionConfig,
success_count: usize,
failure_count: usize,
total_items: usize,
) -> Option<CompletionReason> {
if should_stop_failure(cfg, failure_count, total_items) {
return Some(CompletionReason::FailureToleranceExceeded);
}
if success_count + failure_count < total_items && should_stop_min(cfg, success_count) {
return Some(CompletionReason::MinSuccessfulReached);
}
None
}
fn should_stop_min(
cfg: &crate::builders::map_parallel::CompletionConfig,
success_count: usize,
) -> bool {
cfg.min_successful().is_some_and(|min| success_count >= min)
}
fn should_stop_failure(
cfg: &crate::builders::map_parallel::CompletionConfig,
failure_count: usize,
total_items: usize,
) -> bool {
if !cfg.has_criteria() {
return failure_count > 0;
}
if let Some(tolerated) = cfg.tolerated_failure_count()
&& failure_count > tolerated
{
return true;
}
#[expect(clippy::cast_precision_loss)]
let exceeds = |pct: f64| failure_count as f64 * 100.0 > pct * total_items as f64;
if let Some(pct) = cfg.tolerated_failure_percentage()
&& total_items > 0
&& exceeds(pct)
{
return true;
}
false
}
async fn serialize_value<T, S: Serdes<T>>(
ctx: &DurableContext,
name: Option<&str>,
value: T,
serdes: &S,
serdes_ctx: SerdesContext,
) -> Result<String, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx.hold_activity(serdes.serialize(value, serdes_ctx)).await {
Ok(serialized) => Ok(serialized),
Err(err) => {
let e = ctx
.route_serdes_error("serialize", &operation_id, name, err)
.await;
Err(batch_error(&format!(
"serialize result: {}",
crate::error::wire_message(&*e)
)))
}
}
}
async fn deserialize_value<T, S: Serdes<T>>(
ctx: &DurableContext,
name: Option<&str>,
payload: String,
serdes: &S,
serdes_ctx: SerdesContext,
) -> Result<T, OperationError> {
let operation_id = serdes_ctx.operation_id().to_owned();
match ctx
.hold_activity(serdes.deserialize(payload, serdes_ctx))
.await
{
Ok(value) => Ok(value),
Err(err) => {
let e = ctx
.route_serdes_error("deserialize", &operation_id, name, err)
.await;
Err(batch_error(&format!(
"deserialize result: {}",
crate::error::wire_message(&*e)
)))
}
}
}
fn item_fault_name(item_name: &str) -> Option<&str> {
(!item_name.is_empty()).then_some(item_name)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ItemEncoding {
AllJson,
AllSerdes,
PerItem(Arc<[bool]>),
}
impl ItemEncoding {
fn is_json(&self, index: usize) -> bool {
match self {
Self::AllJson => true,
Self::AllSerdes => false,
#[expect(clippy::expect_used)]
Self::PerItem(flags) => *flags
.get(index)
.expect("parallel_mixed has one encoding flag per branch"),
}
}
}
#[derive(Debug, serde::Serialize)]
struct JsBatchRecord {
all: Vec<JsBatchItem>,
#[serde(rename = "completionReason")]
completion_reason: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct JsBatchItem {
index: usize,
status: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
name: Option<String>,
#[serde(
default,
deserialize_with = "present_raw_value",
skip_serializing_if = "Option::is_none"
)]
result: Option<Box<RawValue>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
error: Option<JsErrorObject>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "PascalCase")]
struct JsErrorObject {
#[serde(default, skip_serializing_if = "Option::is_none")]
error_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_message: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
error_data: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
stack_trace: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cause: Option<Box<JsErrorObject>>,
}
fn present_raw_value<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Box<RawValue>>, D::Error> {
<Box<RawValue> as serde::Deserialize>::deserialize(deserializer).map(Some)
}
#[derive(Debug, Default)]
struct BatchRecordFields {
all: Option<Vec<JsBatchItem>>,
completion_reason: Option<String>,
}
impl<'de> serde::Deserialize<'de> for BatchRecordFields {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
deserializer.deserialize_map(BatchRecordFieldsVisitor)
}
}
struct BatchRecordFieldsVisitor;
impl<'de> serde::de::Visitor<'de> for BatchRecordFieldsVisitor {
type Value = BatchRecordFields;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a batch record object")
}
fn visit_map<A: serde::de::MapAccess<'de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
let mut fields = BatchRecordFields::default();
let mut seen = [false; 2];
let [seen_all, seen_completion_reason] = &mut seen;
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"all" => read_field_once(&mut map, seen_all, "all", &mut fields.all)?,
"completionReason" => read_field_once(
&mut map,
seen_completion_reason,
"completionReason",
&mut fields.completion_reason,
)?,
_ => {
map.next_value::<serde::de::IgnoredAny>()?;
}
}
}
Ok(fields)
}
}
fn read_field_once<'de, A, T>(
map: &mut A,
seen: &mut bool,
name: &'static str,
slot: &mut Option<T>,
) -> Result<(), A::Error>
where
A: serde::de::MapAccess<'de>,
T: serde::Deserialize<'de>,
{
if *seen {
return Err(<A::Error as serde::de::Error>::duplicate_field(name));
}
*seen = true;
*slot = map.next_value()?;
Ok(())
}
fn parse_batch_record(text: &str) -> Result<JsBatchRecord, String> {
let fields: BatchRecordFields =
serde_json::from_str(text).map_err(|err| format!("batch record is not valid: {err}"))?;
match fields {
BatchRecordFields {
all: Some(all),
completion_reason: Some(completion_reason),
} => Ok(JsBatchRecord {
all,
completion_reason,
}),
BatchRecordFields { all: Some(_), .. } => {
Err("batch record has `all` without `completionReason`".to_owned())
}
BatchRecordFields {
completion_reason: Some(_),
..
} => Err("batch record has `completionReason` without `all`".to_owned()),
BatchRecordFields { .. } => {
Err("batch record has neither `all` nor `completionReason`".to_owned())
}
}
}
fn item_status_wire(status: BatchItemStatus) -> &'static str {
match status {
BatchItemStatus::Succeeded => "SUCCEEDED",
BatchItemStatus::Failed => "FAILED",
BatchItemStatus::Started => "STARTED",
}
}
fn parse_item_status(status: &str) -> Result<BatchItemStatus, OperationError> {
match status {
"SUCCEEDED" => Ok(BatchItemStatus::Succeeded),
"FAILED" => Ok(BatchItemStatus::Failed),
"STARTED" => Ok(BatchItemStatus::Started),
other => Err(batch_error(&format!("unknown item status: {other}"))),
}
}
fn js_item_error<O>(item: &BatchItem<O>) -> JsErrorObject {
let message = item.error_message.clone().unwrap_or_default();
let cause = JsErrorObject {
error_type: Some(
item.error_type
.clone()
.unwrap_or_else(|| "Error".to_owned()),
),
error_message: Some(message.clone()),
error_data: item.error_data.clone(),
stack_trace: item.stack_trace.clone(),
cause: None,
};
JsErrorObject {
error_type: Some("ChildContextError".to_owned()),
error_message: Some(message),
error_data: item.error_data.clone(),
stack_trace: item.stack_trace.clone(),
cause: Some(Box::new(cause)),
}
}
fn from_batch_result<O>(
result: &BatchResult<O>,
wires: &[Option<String>],
encoding: &ItemEncoding,
) -> String {
let all = result
.items
.iter()
.zip(wires)
.map(|(item, wire)| {
let value = wire.as_ref().map(|wire| {
let text = if encoding.is_json(item.index) {
wire.clone()
} else {
#[expect(clippy::expect_used)]
serde_json::to_string(wire).expect("a string serializes to JSON")
};
#[expect(clippy::expect_used)]
RawValue::from_string(text).expect("the item value is valid JSON")
});
JsBatchItem {
index: item.index,
status: item_status_wire(item.status).to_owned(),
name: Some(item.name.clone()),
result: value,
error: (item.status == BatchItemStatus::Failed).then(|| js_item_error(item)),
}
})
.collect();
let record = JsBatchRecord {
all,
completion_reason: result.reason.as_str().to_owned(),
};
#[expect(clippy::expect_used)]
serde_json::to_string(&record).expect("the batch record serializes to JSON")
}
fn oversized_batch_summary<O>(
result: &BatchResult<O>,
parent_sub_type: &str,
caller_summary: Option<&str>,
) -> String {
let total = result.items.len();
let succeeded = result.success_count();
let failed = result.failure_count();
let item_statuses: String = result
.items
.iter()
.map(|item| match item.status {
BatchItemStatus::Succeeded => 'S',
BatchItemStatus::Failed => 'F',
BatchItemStatus::Started => '-',
})
.collect();
let status = match result.status() {
BatchStatus::Failed => "FAILED",
BatchStatus::Succeeded => "SUCCEEDED",
};
let parallel = parent_sub_type == PARALLEL_SUB_TYPE;
let summary = OversizedBatchSummary {
type_name: if parallel {
"ParallelResult"
} else {
"MapResult"
},
total_count: total,
success_count: succeeded,
failure_count: failed,
started_count: parallel.then(|| total - succeeded - failed),
completion_reason: result.reason.as_str(),
status,
item_statuses,
summary: caller_summary,
};
#[expect(clippy::expect_used)]
serde_json::to_string(&summary).expect("the oversized batch summary serializes to JSON")
}
#[derive(serde::Serialize)]
#[serde(rename_all = "camelCase")]
struct OversizedBatchSummary<'a> {
#[serde(rename = "type")]
type_name: &'static str,
total_count: usize,
success_count: usize,
failure_count: usize,
#[serde(skip_serializing_if = "Option::is_none")]
started_count: Option<usize>,
completion_reason: &'a str,
status: &'static str,
item_statuses: String,
#[serde(skip_serializing_if = "Option::is_none")]
summary: Option<&'a str>,
}
async fn to_batch_result<O, IS>(
ctx: &DurableContext,
record_text: &str,
serdes: &Arc<IS>,
encoding: ItemEncoding,
total_items: usize,
execution_arn: &str,
item_ids: &ItemIds,
) -> Result<BatchResult<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
let record = parse_batch_record(record_text).map_err(|message| batch_error(&message))?;
js_to_batch_result(
ctx,
record,
serdes,
encoding,
total_items,
execution_arn,
item_ids,
)
.await
}
async fn js_to_batch_result<O, IS>(
ctx: &DurableContext,
record: JsBatchRecord,
serdes: &Arc<IS>,
encoding: ItemEncoding,
total_items: usize,
execution_arn: &str,
item_ids: &ItemIds,
) -> Result<BatchResult<O>, OperationError>
where
O: Send + 'static,
IS: Serdes<O>,
{
if record.all.len() > total_items {
return Err(batch_error(&format!(
"batch record has {} items but the input has {total_items}",
record.all.len()
)));
}
check_record_indexes(&record)?;
let reason = CompletionReason::from_wire(&record.completion_reason);
let mut items = Vec::with_capacity(record.all.len());
for item in record.all {
let index = item.index;
let status = parse_item_status(&item.status)?;
let name = item
.name
.ok_or_else(|| batch_error(&format!("batch record item {index}: missing name")))?;
let result = match (status, item.result) {
(BatchItemStatus::Succeeded, Some(value)) => {
let wire = if encoding.is_json(index) {
value.get().to_owned()
} else {
serde_json::from_str::<String>(value.get()).map_err(|_| {
batch_error(&format!(
"batch record item {index}: expected a string result"
))
})?
};
let child_op_id = item_ids.id(index);
let item_ctx = SerdesContext::new(child_op_id.wire().to_owned(), execution_arn);
Some(
deserialize_value::<O, _>(
ctx,
item_fault_name(&name),
wire,
serdes.as_ref(),
item_ctx,
)
.await?,
)
}
_ => None,
};
let error = item.error.map(|top| match top.cause {
Some(cause) => (
cause.error_type.or(top.error_type),
cause.error_message.or(top.error_message),
cause.error_data.or(top.error_data),
cause.stack_trace,
),
None => (
top.error_type,
top.error_message,
top.error_data,
top.stack_trace,
),
});
let (error_type, error_message, error_data, stack_trace) =
error.unwrap_or((None, None, None, Vec::new()));
items.push(BatchItem {
index,
name,
status,
result,
error_message,
error_type,
error_data,
stack_trace,
});
}
Ok(BatchResult { items, reason })
}
const MAX_CONCURRENCY_ZERO_MESSAGE: &str =
"Invalid max_concurrency: 0. Must be a positive number or unset for unlimited concurrency.";
fn batch_error(message: &str) -> OperationError {
OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
ChildContextErrorKind::Internal,
Some(message.to_owned().into()),
)))
}
fn item_failure_error(wire: crate::error::WireError) -> OperationError {
OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
ChildContextErrorKind::ChildFailed,
Some(crate::error::ReplayedFailure::source_from(wire.clone())),
)))
.with_wire(wire)
}
fn batch_completion_failed_error() -> OperationError {
let wire = crate::error::WireError::new(
Some("BatchCompletionError"),
Some("batch completed as failed by the completion predicate (CUSTOM_COMPLETION_FAILED)"),
);
OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
ChildContextErrorKind::BatchCompletionFailed,
Some(crate::error::ReplayedFailure::source_from(wire.clone())),
)))
.with_wire(wire)
}
fn into_item_slots<I>(items: Vec<I>) -> Arc<Vec<std::sync::Mutex<Option<I>>>> {
Arc::new(
items
.into_iter()
.map(|item| std::sync::Mutex::new(Some(item)))
.collect(),
)
}
fn take_item<I>(items: &[std::sync::Mutex<Option<I>>], index: usize) -> Result<I, OperationError> {
let slot = items
.get(index)
.ok_or_else(|| batch_error("item index out of bounds"))?;
let mut lock = slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
lock.take()
.ok_or_else(|| batch_error("map item already consumed (concurrent access bug)"))
}
#[cfg(test)]
#[expect(clippy::indexing_slicing)]
#[expect(clippy::disallowed_methods)]
mod tests {
use super::*;
use crate::context::DurableContext;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use tokio::sync::Semaphore;
#[test]
fn completion_decision_constructor_builds_complete() {
for expected in [CompletionOutcome::Succeeded, CompletionOutcome::Failed] {
assert!(matches!(
CompletionDecision::complete(expected),
CompletionDecision::Complete(outcome) if outcome == expected
));
}
}
#[test]
fn results_and_errors_are_iterators() {
let batch = BatchResult::new(
vec![
BatchItem::new(
0,
"a".to_owned(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
),
BatchItem::new(
1,
"b".to_owned(),
BatchItemStatus::Failed,
None,
Some("first".to_owned()),
Some("Error".to_owned()),
),
BatchItem::new(
2,
"c".to_owned(),
BatchItemStatus::Started,
None,
None,
None,
),
BatchItem::new(
3,
"d".to_owned(),
BatchItemStatus::Succeeded,
Some(4),
None,
None,
),
BatchItem::new(
4,
"e".to_owned(),
BatchItemStatus::Failed,
None,
Some("second".to_owned()),
Some("TimeoutError".to_owned()),
),
],
CompletionReason::AllCompleted,
);
assert_eq!(batch.results().next(), Some(&1));
assert_eq!(batch.results().copied().collect::<Vec<_>>(), [1, 4]);
let errors = batch
.errors()
.map(|e| (e.index, e.name, e.message, e.error_type))
.collect::<Vec<_>>();
assert_eq!(
errors,
[
(1, "b", "first", Some("Error")),
(4, "e", "second", Some("TimeoutError")),
]
);
}
fn test_ctx(log: CheckpointLog) -> DurableContext {
DurableContext::new_root(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
)
}
#[test]
fn collect_successful_classifies_item_failure_as_child_failed() {
let batch = BatchResult {
items: vec![BatchItem::<i32> {
index: 0,
name: "item-0".to_owned(),
status: BatchItemStatus::Failed,
result: None,
error_message: Some("boom-item".to_owned()),
error_type: Some("MyItemError".to_owned()),
error_data: Some(r#"{"detail":42}"#.to_owned()),
stack_trace: vec!["frame-0".to_owned(), "frame-1".to_owned()],
}],
reason: CompletionReason::FailureToleranceExceeded,
};
let err = collect_successful(batch).expect_err("tolerance exceeded must error");
let OperationErrorKind::ChildContext(child_err) = err.kind() else {
unreachable!("expected a child-context error, got {:?}", err.kind());
};
assert!(
matches!(child_err.kind(), ChildContextErrorKind::ChildFailed),
"item failures classify as ChildFailed: {:?}",
child_err.kind()
);
let wire = err.wire().expect("the item's wire record is attached");
assert_eq!(wire.error_type(), Some("MyItemError"));
assert_eq!(wire.error_message(), Some("boom-item"));
assert_eq!(wire.error_data(), Some(r#"{"detail":42}"#));
assert_eq!(
wire.stack_trace(),
["frame-0".to_owned(), "frame-1".to_owned()]
);
}
#[test]
fn batch_error_keeps_internal_classification_for_coordinator_defects() {
let err = batch_error("invariant breached");
let OperationErrorKind::ChildContext(child_err) = err.kind() else {
unreachable!("expected a child-context error, got {:?}", err.kind());
};
assert!(
matches!(child_err.kind(), ChildContextErrorKind::Internal),
"coordinator defects classify as Internal: {:?}",
child_err.kind()
);
}
fn test_ctx_with_client(
log: CheckpointLog,
) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
(ctx, client)
}
#[tokio::test]
async fn item_hooks_report_container_without_record() {
let hooks = crate::plugin::test_support::OpHooks::default();
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
std::time::Duration::ZERO,
0,
Some(hooks.plugins()),
);
let values = ctx
.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name("m")
.await
.expect("the map succeeds");
assert_eq!(values, [1]);
let map_wire = crate::engine::compute_wire_id_public("1");
let item = hooks.of("map-item-0");
let shape: Vec<_> = item
.iter()
.map(|r| (r.hook, r.parent_id.as_deref()))
.collect();
assert_eq!(
shape,
[
("start", Some(map_wire.as_str())),
("end", Some(map_wire.as_str()))
]
);
}
fn record_at(positional_id: &str, status: CheckpointStatus) -> (String, CheckpointRecord) {
let (wire_id, mut record) = succeeded_record(positional_id, "null");
record.status = status;
record.result = None;
(wire_id, record)
}
#[tokio::test]
async fn normal_item_without_inner_record_replay_flags() {
use crate::plugin::recorder::{UfHooks, context};
let map = |ctx: &DurableContext| {
ctx.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name("m")
};
let hooks = UfHooks::default();
let (ctx, _client) = context(
&hooks,
vec![
record_at("1", CheckpointStatus::Started),
record_at("1-1", CheckpointStatus::Started),
],
);
assert_eq!(map(&ctx).await.unwrap(), [1]);
assert!(hooks.find("map-item-0", "start").replay);
let item = hooks.user_function_of("map-item-0", true);
assert!(item.iter().all(|r| !r.replay), "{item:?}");
let hooks = UfHooks::default();
let (ctx, _client) = context(&hooks, vec![record_at("1", CheckpointStatus::Started)]);
assert_eq!(map(&ctx).await.unwrap(), [1]);
assert!(hooks.find("m", "start").replay);
let container = hooks.user_function_of("m", true);
assert!(container.iter().all(|r| !r.replay), "{container:?}");
}
#[tokio::test]
async fn parallel_mixed_branches_and_container_both_modes() {
use crate::plugin::recorder::{UfHooks, context};
for mode in [NestingMode::Normal, NestingMode::Flat] {
let hooks = UfHooks::default();
let (ctx, _client) = context(&hooks, Vec::new());
let mut set = crate::BranchSet::new();
let text = set.add("s", |_child| async { Ok("text".to_owned()) });
let number = set.add("n", |_child| async { Ok(7_u32) });
ctx.parallel_mixed(set)
.name("pm")
.nesting(mode)
.await
.unwrap();
assert_eq!(text.into_result().unwrap(), "text", "{mode:?}");
assert_eq!(number.into_result().unwrap(), 7, "{mode:?}");
let container_wire = crate::engine::compute_wire_id_public("1");
let container = hooks.user_function_of("pm", true);
assert!(
container
.iter()
.all(|r| { r.sub_type.as_deref() == Some(PARALLEL_SUB_TYPE) && !r.is_virtual }),
"{mode:?}: {container:?}"
);
for name in ["s", "n"] {
let branch = hooks.user_function_of(name, true);
assert!(
branch.iter().all(|r| {
r.sub_type.as_deref() == Some(PARALLEL_BRANCH_SUB_TYPE)
&& r.parent_id.as_deref() == Some(container_wire.as_str())
&& r.is_virtual == (mode == NestingMode::Flat)
}),
"{mode:?} {name}: {branch:?}"
);
assert_eq!(hooks.find(name, "uf_end").ok, Some(true), "{mode:?}");
}
assert_eq!(hooks.find("pm", "uf_end").ok, Some(true), "{mode:?}");
}
}
#[derive(Clone, Copy, Debug)]
enum FailingBatch {
Map,
Parallel,
ParallelMixed,
}
#[tokio::test]
async fn failing_branches_report_stored_wire() {
use crate::builders::map_parallel::CompletionConfig;
use crate::plugin::recorder::{
UfHooks, context, type_and_message, update_error, update_of,
};
let tolerate = || CompletionConfig::with_tolerated_failure_count(1);
let fail = || Err::<u32, BoxError>("boom".into());
for kind in [
FailingBatch::Map,
FailingBatch::Parallel,
FailingBatch::ParallelMixed,
] {
for mode in [NestingMode::Normal, NestingMode::Flat] {
let case = format!("{kind:?} {mode:?}");
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let (failing, kept) = match kind {
FailingBatch::Map => {
let batch = ctx
.map(vec![0_u32, 1], move |_child, item, _idx| async move {
if item == 1 { fail() } else { Ok(item) }
})
.name("c")
.nesting(mode)
.completion(tolerate())
.await_batch()
.await
.unwrap();
let error = batch.errors().next().expect("item 1 failed");
(
"map-item-1",
(
error.error_type.map(str::to_owned),
Some(error.message.to_owned()),
),
)
}
FailingBatch::Parallel => {
let batch = ctx
.parallel(vec![
crate::Branch::new("ok", |_child| async { Ok(0_u32) }),
crate::Branch::new("bad", move |_child| async move { fail() }),
])
.name("c")
.nesting(mode)
.completion(tolerate())
.await_batch()
.await
.unwrap();
let error = batch.errors().next().expect("branch 1 failed");
(
"bad",
(
error.error_type.map(str::to_owned),
Some(error.message.to_owned()),
),
)
}
FailingBatch::ParallelMixed => {
let mut set = crate::BranchSet::new();
let _ok = set.add("ok", |_child| async { Ok(0_u32) });
let bad = set.add("bad", move |_child| async move { fail() });
ctx.parallel_mixed(set)
.name("c")
.nesting(mode)
.completion(tolerate())
.await
.unwrap();
let err = bad.into_result().expect_err("branch 1 failed");
("bad", type_and_message(err.wire().expect("a wire record")))
}
};
let end = hooks.find(failing, "uf_end");
assert_eq!(end.ok, Some(false), "{case}");
let end = type_and_message(end.wire.as_ref().expect("an Err outcome"));
assert_eq!(end.1.as_deref(), Some("boom"), "{case}");
assert_eq!(end, kept, "{case}: the batch keeps the end hook's wire");
let updates = client.recorded_updates();
let item_fail = update_of(&updates, failing, &OperationAction::Fail);
match mode {
NestingMode::Normal => assert_eq!(
update_error(&item_fail.expect("the item's FAIL")),
end,
"{case}: the FAIL stores the end hook's wire"
),
NestingMode::Flat => assert!(item_fail.is_none(), "{case}"),
}
assert_eq!(hooks.find("c", "uf_end").ok, Some(true), "{case}");
assert!(
update_of(&updates, "c", &OperationAction::Succeed).is_some(),
"{case}: the container's SUCCEED"
);
}
}
}
struct FailingResultSerdes;
impl Serdes<String> for FailingResultSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
_value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Err("cannot encode".into())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
_wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Err("cannot decode".into())
}
}
#[derive(Debug)]
struct RejectsParentSucceed(Arc<crate::client::InMemoryExecutionClient>);
impl crate::client::ExecutionClient for RejectsParentSucceed {
fn checkpoint(
&self,
arn: &str,
token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::CheckpointOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
let parent = crate::engine::compute_wire_id_public("1");
if updates
.iter()
.any(|u| u.id == parent && u.action() == &OperationAction::Succeed)
{
return Box::pin(async {
Err(crate::client::ExecutionClientError::new_non_retryable(
"rejected",
))
});
}
self.0.checkpoint(arn, token, updates)
}
fn get_state(
&self,
arn: &str,
token: &str,
marker: Option<&str>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::GetStateOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
self.0.get_state(arn, token, marker)
}
}
#[tokio::test]
async fn container_post_body_failures_keep_own_wire() {
use crate::plugin::recorder::{UfHooks, context, context_over, update_of};
let hooks = UfHooks::default();
let (ctx, client) = context(&hooks, Vec::new());
let err = ctx
.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name("m")
.result_serdes(FailingResultSerdes)
.await
.expect_err("the batch record cannot be serialized");
let end = hooks.find("m", "uf_end");
assert_eq!((end.ok, end.wire), (Some(true), None));
assert_eq!(
err.wire(),
Some(&crate::error::WireError::new(
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE),
Some(
"operation error: child_context: internal error: serialize result: cannot encode"
),
))
);
let updates = client.recorded_updates();
assert!(
update_of(&updates, "m", &OperationAction::Succeed).is_none()
&& update_of(&updates, "m", &OperationAction::Fail).is_none(),
"{updates:?}"
);
let hooks = UfHooks::default();
let inner = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = context_over(
&hooks,
Vec::new(),
Arc::new(RejectsParentSucceed(Arc::clone(&inner))),
);
let outcome = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::driver::test_support::drive_handler(ctx, None, |ctx| async move {
ctx.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name("m")
.await
.map_err(crate::driver::test_support::wire_err)?;
Ok("done".to_owned())
}),
)
.await
.expect("the invocation ends within 5 s")
.expect("no panic");
assert!(
matches!(outcome, crate::driver::InvocationOutcome::Failed { .. }),
"{outcome:?}"
);
let end = hooks.find("m", "uf_end");
assert_eq!((end.ok, end.wire), (Some(true), None));
let updates = inner.recorded_updates();
assert!(
update_of(&updates, "m", &OperationAction::Fail).is_none(),
"no FAIL after the rejected SUCCEED: {updates:?}"
);
assert!(
update_of(&updates, "m", &OperationAction::Start).is_some(),
"the container's START is its last accepted update: {updates:?}"
);
}
#[tokio::test]
async fn abandoned_branch_gets_no_end_hook() {
use crate::builders::map_parallel::CompletionConfig;
use crate::plugin::recorder::{UfHooks, context};
let hooks = UfHooks::default();
let (ctx, _client) = context(&hooks, Vec::new());
let started = Arc::new(tokio::sync::Notify::new());
let waits = Arc::clone(&started);
let batch = tokio::time::timeout(
std::time::Duration::from_secs(5),
ctx.parallel(vec![
crate::Branch::new("done", move |_child| async move {
waits.notified().await;
Ok(0_u32)
}),
crate::Branch::new("abandoned", move |_child| async move {
started.notify_one();
std::future::pending::<()>().await;
Ok(1_u32)
}),
])
.name("p")
.completion(CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch completes within 5 s")
.unwrap();
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
hooks.user_function_of("abandoned", false);
hooks.user_function_of("done", true);
}
fn item_request(
hooks: &crate::plugin::test_support::OpHooks,
nesting: NestingMode,
) -> ItemRequest<crate::serdes::JsonSerdes> {
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let ctx = DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
std::time::Duration::ZERO,
0,
Some(hooks.plugins()),
);
ItemRequest {
ctx,
child_op_id: OperationId::new("1-1".to_owned()),
index: 0,
is_terminal: false,
parent_wire: crate::engine::compute_wire_id_public("1"),
child_sub_type: "MapIteration".to_owned(),
item_name: "map-item-0".to_owned(),
nesting,
start_info: Arc::new(crate::plugin::OperationInfo::for_test("1-1")),
serdes: Arc::new(crate::serdes::JsonSerdes),
start: std::sync::Mutex::new(None),
}
}
#[tokio::test]
async fn flat_item_end_reports_item_context_replay_state() {
for replaying in [true, false] {
let hooks = crate::plugin::test_support::OpHooks::default();
let req = item_request(&hooks, NestingMode::Flat);
item_after(&req, ItemBody::Unhooked(Ok(1_u32)), replaying)
.await
.expect("a flat success settles");
let failed: Result<u32, ChildFnError> = Err(ChildFnError::from(BoxError::from("bad")));
item_after(&req, ItemBody::Unhooked(failed), replaying)
.await
.expect("a flat failure settles");
let ends: Vec<_> = hooks
.of("map-item-0")
.iter()
.map(|r| (r.hook, r.replay, r.is_virtual, r.status))
.collect();
assert_eq!(
ends,
[
(
"end",
replaying,
true,
Some(crate::OperationStatus::Succeeded)
),
("end", replaying, true, Some(crate::OperationStatus::Failed)),
],
"replaying = {replaying}"
);
}
let hooks = crate::plugin::test_support::OpHooks::default();
let req = item_request(&hooks, NestingMode::Normal);
item_ended(&req, crate::OperationStatus::Succeeded, None, None, true);
let normal: Vec<_> = hooks
.of("map-item-0")
.iter()
.map(|r| (r.hook, r.replay, r.is_virtual))
.collect();
assert_eq!(normal, [("end", false, false)]);
}
#[tokio::test]
async fn map_moves_non_json_round_trippable_items() {
use std::collections::HashMap;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let mut m1: HashMap<(u32, u32), u32> = HashMap::new();
m1.insert((1, 2), 10);
let mut m2: HashMap<(u32, u32), u32> = HashMap::new();
m2.insert((3, 4), 20);
m2.insert((5, 6), 30);
let result = ctx
.map(vec![m1, m2], |_child, item, _idx| async move {
Ok(item.values().sum::<u32>())
})
.await
.expect("map over non-JSON-round-trippable items must succeed");
assert_eq!(result, vec![10, 50]);
}
fn child_start_names(client: &crate::client::InMemoryExecutionClient) -> Vec<Option<String>> {
client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Start) && u.parent_id().is_some())
.map(|u| u.name().map(str::to_owned))
.collect()
}
#[tokio::test]
async fn map_item_namer_reaches_child_start_updates() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let result = ctx
.map(
vec![1_i32, 2],
|_child, item, _idx| async move { Ok(item * 10) },
)
.item_namer(|_, i| format!("item-{i}"))
.await
.expect("map must succeed");
assert_eq!(result, vec![10, 20]);
let mut names = child_start_names(&client);
names.sort();
assert_eq!(
names,
vec![Some("item-0".to_owned()), Some("item-1".to_owned())],
"item namer values must be checkpointed on child STARTs"
);
}
#[tokio::test]
async fn parallel_branch_names_reach_child_start_updates() {
use crate::future::Branch;
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let result = ctx
.parallel([
Branch::new("alpha", |_c| async move { Ok(1_i32) }),
Branch::new("beta", |_c| async move { Ok(2_i32) }),
])
.await
.expect("parallel must succeed");
assert_eq!(result, vec![1, 2]);
let mut names = child_start_names(&client);
names.sort();
assert_eq!(
names,
vec![Some("alpha".to_owned()), Some("beta".to_owned())],
"branch names must be checkpointed on child STARTs"
);
}
fn child_update_names(
client: &crate::client::InMemoryExecutionClient,
) -> Vec<(OperationAction, Option<String>)> {
client
.recorded_updates()
.iter()
.filter(|u| u.parent_id().is_some())
.map(|u| (u.action().clone(), u.name().map(str::to_owned)))
.collect()
}
#[tokio::test]
async fn child_terminal_updates_carry_the_recorded_name() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![0_i32, 1], |_child, item, _idx| async move {
if item == 1 {
Err::<i32, BoxError>("intentional".into())
} else {
Ok(item)
}
})
.item_namer(|_, i| {
if i == 0 {
String::new()
} else {
"named".to_owned()
}
})
.max_concurrency(1)
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated item failure must not become an operation error");
assert_eq!(batch.failure_count(), 1);
let updates = child_update_names(&client);
let name_of = |action: OperationAction| -> Vec<Option<String>> {
updates
.iter()
.filter(|(a, _)| *a == action)
.map(|(_, n)| n.clone())
.collect()
};
assert_eq!(
name_of(OperationAction::Succeed),
vec![Some("map-item-0".to_owned())],
"a child SUCCEED must carry the recorded name"
);
assert_eq!(
name_of(OperationAction::Fail),
vec![Some("named".to_owned())],
"a child body-error FAIL must carry the recorded name"
);
}
fn succeeded_record(positional_id: &str, result: &str) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some(result.to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
fn failed_record(positional_id: &str, msg: &str) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Failed,
result: None,
error_type: Some("ChildFnError".to_owned()),
error_message: Some(msg.to_owned()),
error_data: Some(r#"{"recorded":true}"#.to_owned()),
stack_trace: Some(vec!["recorded-frame".to_owned()]),
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
fn replay_children_success_record(positional_id: &str) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: true,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
#[tokio::test]
async fn basic_map_two_items() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let result = ctx
.map(vec![10, 20], |_child, item: i32, _idx| async move {
Ok(item * 2)
})
.await;
assert_eq!(result.unwrap(), vec![20, 40]);
}
#[tokio::test]
async fn basic_parallel_two_branches() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let branches = vec![
Branch::new("a", |_ctx| Box::pin(async { Ok(1) })),
Branch::new("b", |_ctx| Box::pin(async { Ok(2) })),
];
let result = ctx.parallel(branches).await;
assert_eq!(result.unwrap(), vec![1, 2]);
}
#[tokio::test]
async fn parallel_await_batch_reports_per_branch_status_and_reason() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.parallel(vec![
Branch::new("ok-0", |_ctx| async move { Ok(1_i32) }),
Branch::new("boom", |_ctx| async move { Err("intentional".into()) }),
Branch::new("ok-2", |_ctx| async move { Ok(3_i32) }),
])
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated branch failure must not become an operation error");
assert_eq!(batch.reason, CompletionReason::AllCompleted);
assert_eq!(batch.items.len(), 3);
assert_eq!(batch.success_count(), 2);
assert_eq!(batch.failure_count(), 1);
assert_eq!(batch.status(), BatchStatus::Failed);
let by_index = |idx: usize| {
batch
.items
.iter()
.find(|i| i.index == idx)
.expect("batch must contain an item for every started branch")
};
assert_eq!(by_index(0).status, BatchItemStatus::Succeeded);
assert_eq!(by_index(0).result, Some(1));
assert_eq!(by_index(0).name, "ok-0");
assert_eq!(by_index(1).status, BatchItemStatus::Failed);
assert_eq!(by_index(1).result, None);
assert_eq!(by_index(1).name, "boom");
assert!(
by_index(1)
.error_message
.as_deref()
.unwrap_or_default()
.contains("intentional"),
"failed branch must carry its error message: {:?}",
by_index(1).error_message
);
assert_eq!(by_index(2).status, BatchItemStatus::Succeeded);
assert_eq!(by_index(2).result, Some(3));
let errors = batch.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].index, 1);
assert_eq!(errors[0].name, "boom");
assert!(
errors[0].message.contains("intentional"),
"structured error must carry the branch's message: {:?}",
errors[0].message
);
assert_eq!(errors[0].error_type, Some("Error"));
assert_eq!(by_index(1).error_type.as_deref(), Some("Error"));
}
#[tokio::test]
async fn parallel_replayed_terminal_branches_retain_names() {
use crate::future::Branch;
let log = CheckpointLog::from_records(vec![
failed_record("1-1", "recorded failure"),
succeeded_record("1-2", "7"),
]);
let (ctx, _client) = test_ctx_with_client(log);
let batch = ctx
.parallel(vec![
Branch::new("boom", |_ctx| async move { Ok(999_i32) }),
Branch::new("steady", |_ctx| async move { Ok(999_i32) }),
])
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated replayed failure must not become an operation error");
assert_eq!(batch.items.len(), 2);
let by_index = |idx: usize| {
batch
.items
.iter()
.find(|i| i.index == idx)
.expect("batch must contain an item for every branch")
};
let failed = by_index(0);
assert_eq!(failed.status, BatchItemStatus::Failed);
assert_eq!(
failed.name, "boom",
"a replayed failed branch must retain its branch name"
);
assert_eq!(failed.error_message.as_deref(), Some("recorded failure"));
assert_eq!(failed.error_type.as_deref(), Some("ChildFnError"));
assert_eq!(failed.error_data.as_deref(), Some(r#"{"recorded":true}"#));
assert_eq!(failed.stack_trace, ["recorded-frame".to_owned()]);
let succeeded = by_index(1);
assert_eq!(succeeded.status, BatchItemStatus::Succeeded);
assert_eq!(succeeded.result, Some(7));
assert_eq!(
succeeded.name, "steady",
"a replayed succeeded branch must retain its branch name"
);
let errors = batch.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].index, 0);
assert_eq!(
errors[0].name, "boom",
"structured errors must name the replayed failed branch"
);
assert_eq!(errors[0].message, "recorded failure");
assert_eq!(errors[0].error_type, Some("ChildFnError"));
}
fn record_json(record_text: &str) -> serde_json::Value {
serde_json::from_str(record_text).expect("a batch record is JSON")
}
fn recorded_batch_record(client: &crate::client::InMemoryExecutionClient) -> String {
client
.recorded_updates()
.iter()
.filter(|u| u.parent_id().is_none())
.filter_map(|u| u.payload())
.find(|p| parse_batch_record(p).is_ok())
.map(str::to_owned)
.expect("the parent records a batch record payload")
}
#[tokio::test]
async fn map_item_failure_records_body_error_type() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![1_i32], |_child, _item, _idx| async move {
Err::<i32, BoxError>("plain boom".into())
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(0),
)
.await_batch()
.await
.expect("await_batch reports a failed item as a batch result");
let fail = client
.recorded_updates()
.into_iter()
.find(|u| {
matches!(u.action(), OperationAction::Fail)
&& matches!(u.r#type(), OperationType::Context)
})
.expect("the failed item records a child FAIL update");
let error = fail
.error()
.expect("the FAIL update carries an error object");
assert_eq!(error.error_type(), Some("Error"));
assert_eq!(error.error_message(), Some("plain boom"));
let item = batch
.items
.iter()
.find(|i| i.index == 0)
.expect("the batch reports the failed item");
assert_eq!(item.status, BatchItemStatus::Failed);
assert_eq!(item.error_type.as_deref(), Some("Error"));
assert_eq!(
batch.errors().next().map(|e| e.error_type),
Some(Some("Error"))
);
let record = recorded_batch_record(&client);
assert_eq!(
record_json(&record).pointer("/all/0/error/Cause/ErrorType"),
Some(&serde_json::json!("Error"))
);
}
#[tokio::test]
async fn flat_item_failure_records_body_error_type() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![1_i32], |_child, _item, _idx| async move {
Err::<i32, BoxError>("plain boom".into())
})
.nesting(NestingMode::Flat)
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(0),
)
.await_batch()
.await
.expect("await_batch reports a failed item as a batch result");
let item = batch
.items
.iter()
.find(|i| i.index == 0)
.expect("the batch reports the failed item");
assert_eq!(item.status, BatchItemStatus::Failed);
assert_eq!(item.error_type.as_deref(), Some("Error"));
let record = recorded_batch_record(&client);
assert_eq!(
record_json(&record).pointer("/all/0/error/Cause/ErrorType"),
Some(&serde_json::json!("Error"))
);
}
#[tokio::test]
async fn parallel_await_batch_surfaces_tolerance_exceeded_without_error() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.parallel(vec![Branch::new("boom", |_ctx| async move {
Err::<i32, _>("intentional".into())
})])
.completion(
crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_count(0)
.build()
.expect("valid completion config"),
)
.await_batch()
.await
.expect("await_batch must return the batch even when tolerance is exceeded");
assert_eq!(batch.reason, CompletionReason::FailureToleranceExceeded);
assert_eq!(batch.failure_count(), 1);
assert_eq!(
batch.items.first().map(|i| i.status),
Some(BatchItemStatus::Failed)
);
}
#[tokio::test]
async fn parallel_plain_await_tolerates_failures_within_tolerance() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let results: Vec<i32> = ctx
.parallel(vec![
Branch::new("ok-0", |_ctx| async move { Ok(1_i32) }),
Branch::new("boom", |_ctx| async move { Err("intentional".into()) }),
Branch::new("ok-2", |_ctx| async move { Ok(3_i32) }),
])
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await
.expect("a tolerated branch failure must not fail the parallel operation");
assert_eq!(
results,
vec![1, 3],
"successful branch values must be returned in order, tolerated failure omitted"
);
let (map_ctx, _c) = test_ctx_with_client(CheckpointLog::empty());
let map_results: Vec<i32> = map_ctx
.map(vec![1_i32, -1, 3], |_child, item, _idx| async move {
if item < 0 {
Err("intentional".into())
} else {
Ok(item)
}
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await
.expect("map must tolerate the failure");
assert_eq!(
map_results, results,
"map and parallel plain `.await` must agree under the same tolerance"
);
}
#[tokio::test]
async fn absent_completion_config_fails_fast_end_to_end() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let err = ctx
.parallel(vec![
Branch::new("ok", |_ctx| async move { Ok(1_i32) }),
Branch::new("boom", |_ctx| async move { Err("intentional".into()) }),
])
.await
.expect_err("a branch failure without a completion config must fail the batch");
assert!(
format!("{err:#}").contains("intentional"),
"the batch error must carry the failing branch's message, got: {err:#}"
);
let (map_ctx, _c) = test_ctx_with_client(CheckpointLog::empty());
let map_err = map_ctx
.map(vec![1_i32, -1], |_child, item, _idx| async move {
if item < 0 {
Err("intentional".into())
} else {
Ok(item)
}
})
.await
.expect_err("an item failure without a completion config must fail the batch");
assert!(
format!("{map_err:#}").contains("intentional"),
"the batch error must carry the failing item's message, got: {map_err:#}"
);
}
#[tokio::test]
async fn empty_completion_config_fails_fast_end_to_end() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.parallel(vec![Branch::new("boom", |_ctx| async move {
Err::<i32, _>("intentional".into())
})])
.completion(crate::builders::map_parallel::CompletionConfig::default())
.await_batch()
.await
.expect("await_batch must return the batch even when it fails fast");
assert_eq!(
batch.reason,
CompletionReason::FailureToleranceExceeded,
"an empty config must record the same reason as explicit fail-fast"
);
assert_eq!(batch.failure_count(), 1);
let (ctx2, _c2) = test_ctx_with_client(CheckpointLog::empty());
let err = ctx2
.parallel(vec![Branch::new("boom", |_ctx| async move {
Err::<i32, _>("intentional".into())
})])
.completion(crate::builders::map_parallel::CompletionConfig::default())
.await
.expect_err("a failure under an empty config must fail the batch");
assert!(
format!("{err:#}").contains("intentional"),
"the batch error must carry the branch's message, got: {err:#}"
);
}
#[tokio::test]
async fn map_and_parallel_await_batch_agree_on_shape() {
use crate::future::Branch;
let (map_ctx, _c1) = test_ctx_with_client(CheckpointLog::empty());
let map_batch = map_ctx
.map(vec![0_i32, 1, 2], |_child, item, _idx| async move {
if item == 1 {
Err("intentional".into())
} else {
Ok(item * 10)
}
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("map await_batch must tolerate the failure");
let (par_ctx, _c2) = test_ctx_with_client(CheckpointLog::empty());
let par_batch = par_ctx
.parallel((0_i32..3).map(|i| {
Branch::new(format!("branch-{i}"), move |_ctx| async move {
if i == 1 {
Err("intentional".into())
} else {
Ok(i * 10)
}
})
}))
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("parallel await_batch must tolerate the failure");
assert_eq!(map_batch.reason, par_batch.reason);
assert_eq!(map_batch.items.len(), par_batch.items.len());
assert_eq!(map_batch.success_count(), par_batch.success_count());
assert_eq!(map_batch.failure_count(), par_batch.failure_count());
assert_eq!(map_batch.status(), par_batch.status());
let shape = |batch: &BatchResult<i32>| {
let mut items: Vec<_> = batch
.items
.iter()
.map(|i| (i.index, i.status, i.result))
.collect();
items.sort_unstable_by_key(|(idx, _, _)| *idx);
items
};
assert_eq!(
shape(&map_batch),
shape(&par_batch),
"map and parallel batches must report identical per-index outcomes"
);
}
#[tokio::test]
async fn map_and_parallel_accept_into_iterator_and_plain_async() {
use crate::future::Branch;
let ctx = test_ctx(CheckpointLog::empty());
let _ = ctx.map([1_i32, 2, 3], |c, item, _idx| async move {
let v = c.step(move |_| async move { Ok(item + 1) }).await?;
Ok(v)
});
let _ = ctx.map(vec![1_i32, 2], |c, item, _idx| async move {
let v = c.step(move |_| async move { Ok(item) }).await?;
Ok(v)
});
let _ = ctx.map((0_i32..3).map(|x| x * 2), |c, item, _idx| async move {
let v = c.step(move |_| async move { Ok(item) }).await?;
Ok(v)
});
let _ = ctx.parallel([
Branch::new("a", |c| async move {
let v = c.step(|_| async { Ok(1_i32) }).await?;
Ok(v)
}),
Branch::new("b", |c| async move {
let v = c.step(|_| async { Ok(2_i32) }).await?;
Ok(v)
}),
]);
let _ = ctx.parallel(vec![Branch::new("x", |_c| async move { Ok(9_i32) })]);
let _ = ctx.parallel(
(0_i32..2).map(|i| Branch::new(format!("branch-{i}"), move |_c| async move { Ok(i) })),
);
}
#[tokio::test]
async fn replay_frozen_batch_returns_without_re_execution() {
let payload = serde_json::json!({
"all": [
{"index": 0, "name": "map-item-0", "status": "SUCCEEDED", "result": "hello"},
{"index": 1, "name": "map-item-1", "status": "SUCCEEDED", "result": "world"}
],
"completionReason": "ALL_COMPLETED"
});
let log = CheckpointLog::from_records(vec![{
let wire_id = crate::engine::compute_wire_id_public("1");
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some(payload.to_string()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}]);
let ctx = test_ctx(log);
let result: Result<Vec<String>, _> = ctx
.map(
vec!["a".to_owned(), "b".to_owned()],
|_child, _item: String, _idx| async move {
unreachable!("should not execute during replay");
},
)
.await;
let values = result.expect("replay should succeed");
assert_eq!(values, vec!["hello", "world"]);
}
#[tokio::test]
async fn completion_config_min_successful() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.min_successful(2)
.build()
.expect("valid completion config");
assert!(should_stop_min(&cfg, 2));
assert!(should_stop_min(&cfg, 3));
assert!(!should_stop_min(&cfg, 1));
}
#[tokio::test]
async fn completion_config_tolerated_failure_count() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_count(0)
.build()
.expect("valid completion config");
assert!(should_stop_failure(&cfg, 1, 10));
assert!(!should_stop_failure(&cfg, 0, 10));
}
#[tokio::test]
async fn completion_config_tolerated_failure_percentage() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_percentage(20.0)
.build()
.expect("valid completion config");
assert!(should_stop_failure(&cfg, 3, 10));
assert!(!should_stop_failure(&cfg, 2, 10));
}
#[tokio::test]
async fn tolerated_failure_percentage_boundary_cross_multiplication() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_percentage(33.0)
.build()
.expect("valid completion config");
assert!(
should_stop_failure(&cfg, 1, 3),
"1/3 = 33.3% should exceed 33% threshold"
);
assert!(
!should_stop_failure(&cfg, 0, 3),
"0/3 = 0% should not exceed 33%"
);
let cfg34 = crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_percentage(34.0)
.build()
.expect("valid completion config");
assert!(
!should_stop_failure(&cfg34, 1, 3),
"1/3 = 33.3% should not exceed 34%"
);
}
#[tokio::test]
async fn tolerated_failure_percentage_zero_means_fail_fast() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_percentage(0.0)
.build()
.expect("valid completion config");
assert!(
should_stop_failure(&cfg, 1, 10),
"pct=0 should stop on first failure"
);
assert!(
!should_stop_failure(&cfg, 0, 10),
"pct=0 with no failures should not stop"
);
}
#[tokio::test]
async fn empty_completion_config_fails_fast() {
let cfg = crate::builders::map_parallel::CompletionConfig::default();
assert!(
should_stop_failure(&cfg, 1, 10),
"an empty config must fail fast on the first failure"
);
assert!(
!should_stop_failure(&cfg, 0, 10),
"an empty config with no failures must not stop"
);
}
#[tokio::test]
async fn any_configured_criterion_disables_implicit_fail_fast() {
let min_only = crate::builders::map_parallel::CompletionConfig::with_min_successful(2);
assert!(
!should_stop_failure(&min_only, 1, 10),
"a min_successful-only config must tolerate failures"
);
let predicate_only =
crate::builders::map_parallel::CompletionConfig::with_completion_predicate(|_stats| {
CompletionDecision::Continue
});
assert!(
!should_stop_failure(&predicate_only, 1, 10),
"a predicate-only config must tolerate failures"
);
let tolerate_all =
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_percentage(
100.0,
)
.expect("a valid percentage");
assert!(
!should_stop_failure(&tolerate_all, 10, 10),
"pct=100 must tolerate even a fully failed batch"
);
}
#[tokio::test]
async fn max_concurrency_bounds_peak() {
let active = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let active_clone = Arc::clone(&active);
let peak_clone = Arc::clone(&peak);
let sem = Arc::new(Semaphore::new(2));
let mut handles = Vec::new();
for _ in 0..5 {
let sem = Arc::clone(&sem);
let active = Arc::clone(&active_clone);
let peak = Arc::clone(&peak_clone);
handles.push(tokio::spawn(async move {
let _permit = sem.acquire().await.unwrap();
let current = active.fetch_add(1, Ordering::SeqCst) + 1;
loop {
let old = peak.load(Ordering::SeqCst);
if current <= old
|| peak
.compare_exchange(old, current, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
break;
}
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
active.fetch_sub(1, Ordering::SeqCst);
}));
}
for h in handles {
h.await.unwrap();
}
assert!(
peak_clone.load(Ordering::SeqCst) <= 2,
"peak should not exceed max_concurrency=2"
);
}
#[tokio::test]
async fn deterministic_ids_under_concurrent_scheduling() {
let ctx = test_ctx(CheckpointLog::empty());
let id1 = ctx.mint_id();
let id2 = ctx.mint_id();
let id3 = ctx.mint_id();
assert_eq!(id1.positional(), "1");
assert_eq!(id2.positional(), "2");
assert_eq!(id3.positional(), "3");
}
#[tokio::test]
async fn never_started_branches_omitted_from_results() {
let cfg = crate::builders::map_parallel::CompletionConfig::builder()
.min_successful(1)
.build()
.expect("valid completion config");
assert!(should_stop_min(&cfg, 1));
}
use crate::builders::map_parallel::{
CompletionConfig as Config, CompletionDecision, CompletionItem, CompletionOutcome,
};
fn succeed_when(done: bool) -> CompletionDecision {
if done {
CompletionDecision::complete(CompletionOutcome::Succeeded)
} else {
CompletionDecision::Continue
}
}
fn fail_when(done: bool) -> CompletionDecision {
if done {
CompletionDecision::complete(CompletionOutcome::Failed)
} else {
CompletionDecision::Continue
}
}
fn item_is(view: &BatchStats<'_>, index: usize, status: BatchItemStatus) -> bool {
view.items()
.get(index)
.is_some_and(|item| item.status == Some(status))
}
fn parent_payload(client: &crate::client::InMemoryExecutionClient) -> String {
parent_succeed_update(client)
.payload()
.map(str::to_owned)
.expect("the batch parent SUCCEED carries a payload")
}
async fn run_map(
items: Vec<Option<u32>>,
completion: Config,
max_concurrency: usize,
) -> (BatchResult<u32>, String, Result<Vec<u32>, OperationError>) {
let body = |_child: DurableContext, item: Option<u32>, _idx: usize| async move {
item.ok_or_else(|| BoxError::from("boom"))
};
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(items.clone(), body)
.max_concurrency(max_concurrency)
.completion(completion.clone())
.await_batch()
.await
.expect("await_batch never fails for a completed batch");
let payload = parent_payload(&client);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let awaited = ctx
.map(items, body)
.max_concurrency(max_concurrency)
.completion(completion)
.await;
(batch, payload, awaited)
}
fn assert_row(
batch: &BatchResult<u32>,
payload: &str,
reason: CompletionReason,
status: BatchStatus,
) {
assert_eq!(batch.reason, reason);
assert!(
payload.contains(reason.as_str()),
"the parent SUCCEED payload must carry {}, got: {payload}",
reason.as_str()
);
assert_eq!(batch.status(), status);
}
fn wire_message(err: &OperationError) -> Option<String> {
err.wire()
.and_then(crate::error::WireError::error_message)
.map(str::to_owned)
}
fn child_kind(err: &OperationError) -> &ChildContextErrorKind {
let OperationErrorKind::ChildContext(child) = err.kind() else {
unreachable!("expected a child-context error, got {:?}", err.kind());
};
child.kind()
}
#[tokio::test]
async fn min_successful_equal_to_total_records_all_completed() {
let (batch, payload, awaited) =
run_map(vec![Some(1), Some(2)], Config::with_min_successful(2), 1).await;
assert_row(
&batch,
&payload,
CompletionReason::AllCompleted,
BatchStatus::Succeeded,
);
assert_eq!(awaited.expect("Ok"), vec![1, 2]);
}
#[tokio::test]
async fn min_successful_before_last_item_records_min_successful() {
let (batch, payload, awaited) = run_map(
vec![Some(1), Some(2), Some(3)],
Config::with_min_successful(1),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::MinSuccessfulReached,
BatchStatus::Succeeded,
);
assert_eq!(batch.items.len(), 1);
assert_eq!(awaited.expect("Ok"), vec![1]);
}
#[tokio::test]
async fn tolerance_exceeded_on_last_item_records_failure() {
let (batch, payload, awaited) = run_map(
vec![Some(1), None],
Config::with_tolerated_failure_count(0),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::FailureToleranceExceeded,
BatchStatus::Failed,
);
let err = awaited.expect_err("the tolerance is exceeded");
assert!(matches!(
child_kind(&err),
ChildContextErrorKind::ChildFailed
));
assert_eq!(wire_message(&err).as_deref(), Some("boom"));
}
#[tokio::test]
async fn min_not_reached_records_all_completed() {
let (batch, payload, awaited) = run_map(
vec![Some(1), None, Some(3)],
Config::with_min_successful(3),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::AllCompleted,
BatchStatus::Failed,
);
assert_eq!(awaited.expect("Ok"), vec![1, 3]);
}
#[tokio::test]
async fn predicate_complete_succeeded() {
let (batch, payload, awaited) = run_map(
vec![Some(1), Some(2), Some(3)],
Config::with_completion_predicate(|s| succeed_when(s.succeeded() >= 1)),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::CustomCompletionSucceeded,
BatchStatus::Succeeded,
);
assert_eq!(batch.items.len(), 1);
assert_eq!(awaited.expect("Ok"), vec![1]);
}
#[tokio::test]
async fn predicate_complete_failed_without_failed_item() {
let (batch, payload, awaited) = run_map(
vec![Some(1), Some(2)],
Config::with_completion_predicate(|s| fail_when(s.settled() >= 1)),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::CustomCompletionFailed,
BatchStatus::Failed,
);
assert_eq!(batch.items.len(), 1);
let err = awaited.expect_err("the predicate failed the batch");
assert!(matches!(
child_kind(&err),
ChildContextErrorKind::BatchCompletionFailed
));
let wire = err.wire().expect("a wire record is attached");
assert_eq!(wire.error_type(), Some("BatchCompletionError"));
assert_eq!(
wire.error_message(),
Some(
"batch completed as failed by the completion predicate (CUSTOM_COMPLETION_FAILED)"
)
);
assert_eq!(child_kind(&err).to_string(), "batch completed as failed");
}
#[tokio::test]
async fn predicate_complete_failed_with_failed_item() {
let (batch, payload, awaited) = run_map(
vec![None, Some(2)],
Config::with_completion_predicate(|s| fail_when(s.failed() >= 1)),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::CustomCompletionFailed,
BatchStatus::Failed,
);
assert_eq!(batch.items.len(), 1);
let err = awaited.expect_err("the predicate failed the batch");
assert!(matches!(
child_kind(&err),
ChildContextErrorKind::ChildFailed
));
assert_eq!(wire_message(&err).as_deref(), Some("boom"));
}
#[tokio::test]
async fn predicate_continue_records_all_completed() {
let (batch, payload, awaited) = run_map(
vec![Some(1), None],
Config::with_completion_predicate(|_| CompletionDecision::Continue),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::AllCompleted,
BatchStatus::Failed,
);
assert_eq!(awaited.expect("Ok"), vec![1]);
}
#[tokio::test(start_paused = true)]
async fn first_trigger_reason_wins_while_a_later_item_runs() {
let body = |_child: DurableContext, item: Option<u32>, _idx: usize| async move {
match item {
Some(v) => {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
Ok(v)
}
None => Err(BoxError::from("boom")),
}
};
let completion = Config::builder()
.min_successful(1)
.tolerated_failure_count(0)
.build()
.expect("valid completion config");
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![None, Some(2)], body)
.max_concurrency(2)
.completion(completion.clone())
.await_batch()
.await
.expect("await_batch returns the batch");
assert_row(
&batch,
&parent_payload(&client),
CompletionReason::FailureToleranceExceeded,
BatchStatus::Failed,
);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let err = ctx
.map(vec![None, Some(2)], body)
.max_concurrency(2)
.completion(completion)
.await
.expect_err("the tolerance is exceeded");
assert!(matches!(
child_kind(&err),
ChildContextErrorKind::ChildFailed
));
assert_eq!(wire_message(&err).as_deref(), Some("boom"));
}
#[tokio::test]
async fn predicate_complete_on_last_settle_records_custom() {
let (batch, payload, awaited) = run_map(
vec![Some(1), Some(2)],
Config::with_completion_predicate(|s| succeed_when(s.settled() == s.total_items())),
1,
)
.await;
assert_row(
&batch,
&payload,
CompletionReason::CustomCompletionSucceeded,
BatchStatus::Succeeded,
);
assert_eq!(awaited.expect("Ok"), vec![1, 2]);
}
fn counting_body(
counters: Arc<Vec<AtomicUsize>>,
) -> impl Fn(DurableContext, usize, usize) -> std::future::Ready<Result<usize, BoxError>>
+ Send
+ Sync
+ 'static {
move |_child, item, _idx| {
if let Some(counter) = counters.get(item) {
counter.fetch_add(1, Ordering::SeqCst);
}
std::future::ready(Ok(item))
}
}
fn counters(n: usize) -> Arc<Vec<AtomicUsize>> {
Arc::new((0..n).map(|_| AtomicUsize::new(0)).collect())
}
fn counts(counters: &[AtomicUsize]) -> Vec<usize> {
counters.iter().map(|c| c.load(Ordering::SeqCst)).collect()
}
#[tokio::test]
async fn predicate_complete_before_first_dispatch_runs_no_body() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let ran = counters(2);
let batch = ctx
.map(vec![0_usize, 1], counting_body(Arc::clone(&ran)))
.completion(Config::with_completion_predicate(|_| succeed_when(true)))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(counts(&ran), [0, 0]);
assert!(batch.items.is_empty());
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
let parent_succeeds = client
.recorded_updates()
.iter()
.filter(|u| u.parent_id().is_none() && matches!(u.action(), OperationAction::Succeed))
.count();
assert_eq!(parent_succeeds, 1);
}
#[tokio::test]
async fn predicate_complete_at_dispatch_gate_stops_later_starts() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let ran = counters(3);
let batch = ctx
.map(vec![0_usize, 1, 2], counting_body(Arc::clone(&ran)))
.max_concurrency(1)
.completion(Config::with_completion_predicate(|s| {
succeed_when(item_is(s, 0, BatchItemStatus::Succeeded))
}))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(counts(&ran), [1, 0, 0]);
assert_eq!(statuses(&batch), [(0, BatchItemStatus::Succeeded)]);
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
}
#[tokio::test]
async fn empty_batch_predicate_failed_records_custom_failed() {
let completion = Config::with_completion_predicate(|_| fail_when(true));
let body = |_child: DurableContext, item: u32, _idx: usize| async move { Ok(item) };
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(Vec::<u32>::new(), body)
.completion(completion.clone())
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(batch.reason, CompletionReason::CustomCompletionFailed);
assert_eq!(batch.status(), BatchStatus::Failed);
assert!(parent_payload(&client).contains("CUSTOM_COMPLETION_FAILED"));
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let err = ctx
.map(Vec::<u32>::new(), body)
.completion(completion)
.await
.expect_err("the predicate failed the empty batch");
assert!(matches!(
child_kind(&err),
ChildContextErrorKind::BatchCompletionFailed
));
}
#[tokio::test]
async fn empty_batch_predicate_continue_records_all_completed() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let calls = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&calls);
let batch = ctx
.map(
Vec::<u32>::new(),
|_child, item, _idx| async move { Ok(item) },
)
.completion(Config::with_completion_predicate(move |s| {
seen.fetch_add(1, Ordering::SeqCst);
assert_eq!(s.total_items(), 0);
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(batch.reason, CompletionReason::AllCompleted);
assert_eq!(calls.load(Ordering::SeqCst), 1, "asked exactly once");
}
#[tokio::test]
async fn min_successful_zero_one_item_records_all_completed() {
let (batch, _payload, awaited) =
run_map(vec![None], Config::with_min_successful(0), 1).await;
assert_eq!(batch.reason, CompletionReason::AllCompleted);
assert_eq!(batch.status(), BatchStatus::Failed);
assert_eq!(awaited.expect("Ok"), Vec::<u32>::new());
}
#[tokio::test]
async fn min_successful_zero_completes_at_first_settle() {
let ran = Arc::new(AtomicUsize::new(0));
let item_one_ran = Arc::clone(&ran);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![None, Some(1_u32)], move |_child, item, _idx| {
let item_one_ran = Arc::clone(&item_one_ran);
async move {
match item {
Some(v) => {
item_one_ran.fetch_add(1, Ordering::SeqCst);
Ok(v)
}
None => Err(BoxError::from("boom")),
}
}
})
.max_concurrency(1)
.completion(Config::with_min_successful(0))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(statuses(&batch), [(0, BatchItemStatus::Failed)]);
assert_eq!(ran.load(Ordering::SeqCst), 0);
}
async fn percentage_reason(total: usize, failures: usize, pct: f64) -> CompletionReason {
let items: Vec<Option<u32>> = (0..total)
.map(|i| if i < failures { None } else { Some(1) })
.collect();
let (batch, _payload, _awaited) = run_map(
items,
Config::with_tolerated_failure_percentage(pct).expect("a valid percentage"),
1,
)
.await;
batch.reason
}
#[tokio::test]
async fn percentage_boundary_is_exact() {
assert_ne!(
percentage_reason(50, 7, 14.0).await,
CompletionReason::FailureToleranceExceeded
);
assert_eq!(
percentage_reason(50, 8, 14.0).await,
CompletionReason::FailureToleranceExceeded
);
}
#[tokio::test]
async fn fractional_percentage() {
assert_ne!(
percentage_reason(8, 1, 12.5).await,
CompletionReason::FailureToleranceExceeded
);
assert_eq!(
percentage_reason(8, 2, 12.5).await,
CompletionReason::FailureToleranceExceeded
);
}
type ViewLog = Arc<std::sync::Mutex<Vec<Vec<(Option<String>, Option<BatchItemStatus>)>>>>;
fn view_of(stats: &BatchStats<'_>) -> Vec<(Option<String>, Option<BatchItemStatus>)> {
stats
.items()
.iter()
.map(|i| (i.name.clone(), i.status))
.collect()
}
fn named(
entries: &[(&str, Option<BatchItemStatus>)],
) -> Vec<(Option<String>, Option<BatchItemStatus>)> {
entries
.iter()
.map(|(name, status)| (Some((*name).to_owned()), *status))
.collect()
}
fn ok_branch(name: &str, value: u32) -> crate::future::Branch<u32> {
crate::future::Branch::new(name, move |_ctx| async move { Ok(value) })
}
fn pending_branch(name: &str) -> crate::future::Branch<u32> {
crate::future::Branch::new(name, |ctx| async move {
Ok(ctx
.step(|_| std::future::pending::<Result<u32, BoxError>>())
.name("forever")
.await?)
})
}
#[tokio::test]
async fn predicate_completes_on_later_items_before_a_slow_earlier_item() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.parallel(vec![
pending_branch("slow"),
ok_branch("b", 2),
ok_branch("c", 3),
])
.completion(Config::with_completion_predicate(|s| {
succeed_when(
item_is(s, 1, BatchItemStatus::Succeeded)
&& item_is(s, 2, BatchItemStatus::Succeeded),
)
}))
.await_batch(),
)
.await
.expect("the predicate does not wait for item 0")
.expect("await_batch returns the batch");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Started),
(1, BatchItemStatus::Succeeded),
(2, BatchItemStatus::Succeeded)
]
);
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
}
#[tokio::test]
async fn predicate_sees_names_and_not_started_items() {
let log: ViewLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
ctx.parallel(vec![
ok_branch("a", 1),
ok_branch("b", 2),
ok_branch("c", 3),
])
.max_concurrency(1)
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
let ok = Some(BatchItemStatus::Succeeded);
let first = named(&[("a", None), ("b", None), ("c", None)]);
let after_a = named(&[("a", ok), ("b", None), ("c", None)]);
let after_b = named(&[("a", ok), ("b", ok), ("c", None)]);
let after_c = named(&[("a", ok), ("b", ok), ("c", ok)]);
assert_eq!(
*log.lock().expect("no poisoned lock"),
vec![
first,
after_a.clone(),
after_a,
after_b.clone(),
after_b,
after_c
]
);
}
#[tokio::test]
async fn predicate_sees_a_running_item_as_started() {
let log: ViewLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.parallel(vec![ok_branch("a", 1), pending_branch("b")])
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
succeed_when(s.succeeded() >= 1)
}))
.await_batch(),
)
.await
.expect("the batch does not wait for item b")
.expect("await_batch returns the batch");
assert_eq!(
log.lock().expect("no poisoned lock").last().cloned(),
Some(named(&[
("a", Some(BatchItemStatus::Succeeded)),
("b", Some(BatchItemStatus::Started))
]))
);
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
}
#[tokio::test]
async fn predicate_counts_follow_settle_order() {
type CountLog = Arc<std::sync::Mutex<Vec<(usize, usize, usize)>>>;
let log: CountLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
tokio::time::timeout(
EARLY_BOUND,
ctx.parallel(vec![
pending_branch("slow"),
ok_branch("b", 2),
ok_branch("c", 3),
])
.completion(Config::with_completion_predicate(move |s| {
if s.settled() > 0 {
writer.lock().expect("no poisoned lock").push((
s.succeeded(),
s.failed(),
s.settled(),
));
}
succeed_when(s.settled() == 2)
}))
.await_batch(),
)
.await
.expect("the counts do not wait for item 0")
.expect("await_batch returns the batch");
assert_eq!(
*log.lock().expect("no poisoned lock"),
vec![(1, 0, 1), (2, 0, 2)]
);
}
#[tokio::test]
async fn resumed_batch_keeps_recorded_outcomes() {
let log = CheckpointLog::from_records(vec![
succeeded_record("1-1", "1"),
succeeded_record("1-2", "2"),
failed_record("1-3", "recorded failure"),
]);
let (ctx, _client) = test_ctx_with_client(log);
let ran = Arc::new(AtomicUsize::new(0));
let body_ran = Arc::clone(&ran);
let batch = ctx
.map(vec![0_u32, 1, 2], move |_child, _item, _idx| {
body_ran.fetch_add(1, Ordering::SeqCst);
async move { Ok(999_u32) }
})
.max_concurrency(2)
.completion(Config::with_completion_predicate(|s| {
succeed_when(s.succeeded() >= 1)
}))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Succeeded),
(2, BatchItemStatus::Failed)
]
);
assert_eq!(batch.results().collect::<Vec<_>>(), [&1, &2]);
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
assert_eq!(ran.load(Ordering::SeqCst), 0, "no body ran");
}
#[test]
fn first_threshold_trigger_wins_across_settles() {
let cfg = Config::builder()
.min_successful(1)
.tolerated_failure_count(0)
.build()
.expect("valid completion config");
let mut tracker = CompletionTracker::new(vec![None; 3]);
tracker.settle(&cfg, 3, 0, BatchItemStatus::Failed);
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::FailureToleranceExceeded)
);
tracker.settle(&cfg, 3, 1, BatchItemStatus::Succeeded);
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::FailureToleranceExceeded)
);
let mut tracker = CompletionTracker::new(vec![None; 3]);
tracker.settle(&cfg, 3, 0, BatchItemStatus::Succeeded);
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::MinSuccessfulReached)
);
tracker.settle(&cfg, 3, 1, BatchItemStatus::Failed);
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::MinSuccessfulReached)
);
}
#[test]
fn stopped_tracker_settles_without_evaluating() {
let calls = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&calls);
let predicate = Config::with_completion_predicate(move |_| {
seen.fetch_add(1, Ordering::SeqCst);
CompletionDecision::complete(CompletionOutcome::Failed)
});
let threshold = Config::with_tolerated_failure_count(0);
for cfg in [&predicate, &threshold] {
let mut tracker = CompletionTracker::new(vec![None; 2]);
tracker.stop_evaluating();
tracker.settle(cfg, 2, 0, BatchItemStatus::Failed);
tracker.settle(cfg, 2, 1, BatchItemStatus::Succeeded);
assert_eq!(tracker.stop_reason, None);
assert_eq!((tracker.success_count, tracker.failure_count), (1, 1));
assert_eq!(
tracker
.items
.iter()
.map(|item| item.status)
.collect::<Vec<_>>(),
[
Some(BatchItemStatus::Failed),
Some(BatchItemStatus::Succeeded)
]
);
}
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"the predicate is never called"
);
}
#[test]
fn stopped_tracker_finish_replay_pass_checks_nothing() {
let cfg = Config::builder()
.min_successful(1)
.tolerated_failure_count(0)
.build()
.expect("valid completion config");
let mut tracker = CompletionTracker::new(vec![None; 2]);
tracker.stop_evaluating();
tracker.replaying_recorded = true;
tracker.settle(&cfg, 2, 0, BatchItemStatus::Succeeded);
tracker.settle(&cfg, 2, 1, BatchItemStatus::Failed);
tracker.finish_replay_pass(&cfg, 2);
assert_eq!(tracker.stop_reason, None);
assert!(!tracker.replaying_recorded);
}
#[test]
fn first_predicate_decision_wins_across_settles() {
let calls = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&calls);
let cfg = Config::with_completion_predicate(move |s| {
seen.fetch_add(1, Ordering::SeqCst);
match s.settled() {
1 => CompletionDecision::complete(CompletionOutcome::Succeeded),
2 => CompletionDecision::complete(CompletionOutcome::Failed),
_ => CompletionDecision::Continue,
}
});
let mut tracker = CompletionTracker::new(vec![None; 3]);
tracker.settle(&cfg, 3, 0, BatchItemStatus::Succeeded);
tracker.settle(&cfg, 3, 1, BatchItemStatus::Succeeded);
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::CustomCompletionSucceeded)
);
assert_eq!(calls.load(Ordering::SeqCst), 1, "the predicate ran once");
assert!(tracker.dispatch_gate(&cfg));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[test]
fn batch_stats_new_counts_items() {
let items = [
CompletionItem::new(0, None, Some(BatchItemStatus::Succeeded)),
CompletionItem::new(1, None, Some(BatchItemStatus::Failed)),
CompletionItem::new(2, None, Some(BatchItemStatus::Started)),
CompletionItem::new(3, None, None),
];
let stats = BatchStats::new(&items);
assert_eq!(stats.succeeded(), 1);
assert_eq!(stats.failed(), 1);
assert_eq!(stats.settled(), 2);
assert_eq!(stats.total_items(), 4);
assert_eq!(stats.items().len(), 4);
}
const SETTLE_ORDERS: [[usize; 3]; 6] = [
[0, 1, 2],
[0, 2, 1],
[1, 0, 2],
[1, 2, 0],
[2, 0, 1],
[2, 1, 0],
];
fn abc() -> Vec<Option<String>> {
vec![Some("a".into()), Some("b".into()), Some("c".into())]
}
#[test]
fn monotonic_predicate_records_the_same_outcome_in_every_settle_order() {
for order in SETTLE_ORDERS {
let completes = Arc::new(AtomicUsize::new(0));
let seen = Arc::clone(&completes);
let cfg = Config::with_completion_predicate(move |s| {
let decision = succeed_when(
item_is(s, 1, BatchItemStatus::Succeeded)
&& item_is(s, 2, BatchItemStatus::Succeeded),
);
if decision != CompletionDecision::Continue {
seen.fetch_add(1, Ordering::SeqCst);
}
decision
});
let mut tracker = CompletionTracker::new(abc());
for index in order {
tracker.settle(&cfg, 3, index, BatchItemStatus::Succeeded);
}
assert_eq!(
tracker.stop_reason,
Some(CompletionReason::CustomCompletionSucceeded),
"order {order:?}"
);
assert_eq!(completes.load(Ordering::SeqCst), 1, "order {order:?}");
}
}
#[test]
fn non_monotonic_predicate_depends_on_settle_order() {
let cfg = Config::with_completion_predicate(|s| {
succeed_when(s.succeeded() == 1 && s.failed() == 0)
});
let mut order_a = CompletionTracker::new(abc());
order_a.settle(&cfg, 3, 0, BatchItemStatus::Succeeded);
order_a.settle(&cfg, 3, 1, BatchItemStatus::Failed);
assert_eq!(
order_a.stop_reason,
Some(CompletionReason::CustomCompletionSucceeded)
);
let mut order_b = CompletionTracker::new(abc());
order_b.settle(&cfg, 3, 1, BatchItemStatus::Failed);
order_b.settle(&cfg, 3, 0, BatchItemStatus::Succeeded);
assert_eq!(order_b.stop_reason, None);
}
#[test]
fn tracker_marks_dispatched_items_started() {
let cfg = Config::default();
let mut tracker = CompletionTracker::new(vec![Some("a".into()), Some("b".into())]);
let view = |tracker: &CompletionTracker| -> Vec<Option<BatchItemStatus>> {
tracker.items.iter().map(|i| i.status).collect()
};
assert_eq!(view(&tracker), [None, None]);
tracker.mark_dispatched(0);
assert_eq!(view(&tracker), [Some(BatchItemStatus::Started), None]);
tracker.settle(&cfg, 2, 0, BatchItemStatus::Succeeded);
assert_eq!(view(&tracker), [Some(BatchItemStatus::Succeeded), None]);
tracker.mark_dispatched(1);
assert_eq!(
view(&tracker),
[
Some(BatchItemStatus::Succeeded),
Some(BatchItemStatus::Started)
]
);
assert_eq!(
tracker.items[1].name.as_deref(),
Some("b"),
"the view keeps the caller name"
);
}
#[tokio::test]
async fn predicate_sees_map_namer_names() {
let log: ViewLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
ctx.map(vec![0_u32, 1], |_child, item, _idx| async move { Ok(item) })
.item_namer(|_, i| {
if i == 0 {
String::new()
} else {
format!("n{i}")
}
})
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
let names: Vec<Option<String>> = log.lock().expect("no poisoned lock")[0]
.iter()
.map(|(name, _)| name.clone())
.collect();
assert_eq!(names, [Some(String::new()), Some("n1".to_owned())]);
let log: ViewLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
ctx.map(vec![0_u32], |_child, item, _idx| async move { Ok(item) })
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(log.lock().expect("no poisoned lock")[0], [(None, None)]);
}
#[tokio::test]
async fn item_namer_runs_once_per_item_and_names_agree() {
for concurrency in [1, 3] {
let calls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&calls);
let log: ViewLog = Arc::default();
let writer = Arc::clone(&log);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(
vec![0_u32, 1, 2],
|_child, item, _idx| async move { Ok(item) },
)
.item_namer(move |_, i| {
let call = counter.fetch_add(1, Ordering::SeqCst);
format!("n{i}-call{call}")
})
.max_concurrency(concurrency)
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(
calls.load(Ordering::SeqCst),
3,
"the namer runs once per item (concurrency {concurrency})"
);
let result_names: Vec<Option<String>> =
batch.items.iter().map(|i| Some(i.name.clone())).collect();
let view_names: Vec<Option<String>> = log
.lock()
.expect("no poisoned lock")
.last()
.expect("the predicate ran")
.iter()
.map(|(name, _)| name.clone())
.collect();
let mut start_names = child_start_names(&client);
start_names.sort();
assert_eq!(view_names, result_names, "concurrency {concurrency}");
assert_eq!(start_names, result_names, "concurrency {concurrency}");
}
}
struct BatchNames {
starts: Vec<Option<String>>,
results: Vec<String>,
predicate: Vec<Option<String>>,
}
impl BatchNames {
fn of<O>(
client: &crate::client::InMemoryExecutionClient,
batch: &BatchResult<O>,
log: &ViewLog,
) -> Self {
Self {
starts: child_start_names(client),
results: batch.items.iter().map(|i| i.name.clone()).collect(),
predicate: log.lock().expect("no poisoned lock")[0]
.iter()
.map(|(name, _)| name.clone())
.collect(),
}
}
}
fn logging_predicate(log: &ViewLog) -> Config {
let writer = Arc::clone(log);
Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
})
}
fn some(names: &[&str]) -> Vec<Option<String>> {
names.iter().map(|n| Some((*n).to_owned())).collect()
}
fn owned(names: &[&str]) -> Vec<String> {
names.iter().map(|n| (*n).to_owned()).collect()
}
#[tokio::test]
async fn map_item_namer_reads_the_item() {
struct Order {
id: String,
}
let log: ViewLog = Arc::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let orders = vec![
Order {
id: "o-1".to_owned(),
},
Order {
id: "o-2".to_owned(),
},
];
let batch = ctx
.map(orders, |_child, order: Order, _idx| async move {
Ok(order.id.len())
})
.item_namer(|o, _| o.id.clone())
.max_concurrency(1)
.completion(logging_predicate(&log))
.await_batch()
.await
.expect("await_batch returns the batch");
let names = BatchNames::of(&client, &batch, &log);
assert_eq!(names.starts, some(&["o-1", "o-2"]));
assert_eq!(names.results, owned(&["o-1", "o-2"]));
assert_eq!(names.predicate, some(&["o-1", "o-2"]));
}
#[tokio::test]
async fn unnamed_map_items_record_default_names() {
let log: ViewLog = Arc::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![1_u32, 2], |_child, item, _idx| async move { Ok(item) })
.max_concurrency(1)
.completion(logging_predicate(&log))
.await_batch()
.await
.expect("await_batch returns the batch");
let names = BatchNames::of(&client, &batch, &log);
assert_eq!(names.starts, some(&["map-item-0", "map-item-1"]));
assert_eq!(names.results, owned(&["map-item-0", "map-item-1"]));
assert_eq!(names.predicate, [None, None]);
}
#[tokio::test]
async fn empty_namer_result_records_default_name() {
let log: ViewLog = Arc::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![1_u32, 2], |_child, item, _idx| async move { Ok(item) })
.item_namer(|_, i| {
if i == 0 {
String::new()
} else {
"named".into()
}
})
.max_concurrency(1)
.completion(logging_predicate(&log))
.await_batch()
.await
.expect("await_batch returns the batch");
let names = BatchNames::of(&client, &batch, &log);
assert_eq!(names.starts, some(&["map-item-0", "named"]));
assert_eq!(names.results, owned(&["map-item-0", "named"]));
assert_eq!(names.predicate, some(&["", "named"]));
}
#[tokio::test]
async fn empty_branch_name_records_default_name() {
use crate::future::Branch;
let log: ViewLog = Arc::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.parallel([
Branch::new("", |_c| async move { Ok(1_u32) }),
Branch::new("x", |_c| async move { Ok(2_u32) }),
])
.max_concurrency(1)
.completion(logging_predicate(&log))
.await_batch()
.await
.expect("await_batch returns the batch");
let names = BatchNames::of(&client, &batch, &log);
assert_eq!(names.starts, some(&["parallel-branch-0", "x"]));
assert_eq!(names.results, owned(&["parallel-branch-0", "x"]));
assert_eq!(names.predicate, some(&["", "x"]));
}
#[tokio::test]
async fn unnamed_branch_records_default_name() {
use crate::future::Branch;
let log: ViewLog = Arc::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.parallel([
Branch::unnamed(|_c| async move { Ok(1_u32) }),
Branch::new("x", |_c| async move { Ok(2_u32) }),
])
.max_concurrency(1)
.completion(logging_predicate(&log))
.await_batch()
.await
.expect("await_batch returns the batch");
let names = BatchNames::of(&client, &batch, &log);
assert_eq!(names.starts, some(&["parallel-branch-0", "x"]));
assert_eq!(names.results, owned(&["parallel-branch-0", "x"]));
assert_eq!(names.predicate, [None, Some("x".to_owned())]);
}
#[tokio::test]
async fn namer_runs_once_per_item() {
let calls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&calls);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
ctx.map(
vec![1_u32, 2, 3],
|_child, item, _idx| async move { Ok(item) },
)
.item_namer(move |_, _| {
counter.fetch_add(1, Ordering::SeqCst);
String::new()
})
.max_concurrency(1)
.completion(Config::with_min_successful(1))
.await_batch()
.await
.expect("await_batch returns the batch");
assert_eq!(child_start_names(&client), some(&["map-item-0"]));
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn sequential_replay_asks_no_gate_for_a_recorded_item() {
let log = CheckpointLog::from_records(vec![succeeded_record("1-1", "1")]);
let (ctx, client) = test_ctx_with_client(log);
let views: ViewLog = Arc::default();
let writer = Arc::clone(&views);
let ran = Arc::new(AtomicUsize::new(0));
let body_ran = Arc::clone(&ran);
let batch = ctx
.map(vec![0_u32, 1, 2], move |_child, item, _idx| {
body_ran.fetch_add(1, Ordering::SeqCst);
async move { Ok(item + 10) }
})
.max_concurrency(1)
.completion(Config::with_completion_predicate(move |s| {
writer.lock().expect("no poisoned lock").push(view_of(s));
CompletionDecision::Continue
}))
.await_batch()
.await
.expect("await_batch returns the batch");
let ok = Some(BatchItemStatus::Succeeded);
let after_0 = vec![(None, ok), (None, None), (None, None)];
let after_1 = vec![(None, ok), (None, ok), (None, None)];
let after_2 = vec![(None, ok), (None, ok), (None, ok)];
assert_eq!(
*views.lock().expect("no poisoned lock"),
vec![after_0.clone(), after_0, after_1.clone(), after_1, after_2],
"settle of item 0, gate and settle of item 1, gate and settle of item 2"
);
assert_eq!(batch.results().collect::<Vec<_>>(), [&1, &11, &12]);
assert_eq!(ran.load(Ordering::SeqCst), 2, "only the live items ran");
let started: Vec<String> = client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Start) && u.parent_id().is_some())
.map(|u| u.id().to_owned())
.collect();
assert_eq!(
started,
[
crate::engine::compute_wire_id_public("1-2"),
crate::engine::compute_wire_id_public("1-3")
],
"the live items keep the IDs the first run gives them"
);
}
#[tokio::test]
async fn completion_predicate_ends_map_batch_early() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(
vec![10_u32, 20, 30, 40, 50],
|_child, item, _idx| async move { Ok(item) },
)
.name("predicated")
.max_concurrency(1)
.completion(
crate::builders::map_parallel::CompletionConfig::builder()
.completion_predicate(|stats| succeed_when(stats.settled() >= 2))
.build()
.expect("valid completion config"),
)
.await_batch()
.await
.expect("a predicate-completed batch must not become an operation error");
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
assert_eq!(
batch.items.len(),
2,
"the predicate fires after the second settle; later items never start"
);
assert_eq!(batch.success_count(), 2);
let parent_success_payload = client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Succeed) && u.parent_id().is_none())
.filter_map(|u| u.payload().map(str::to_owned))
.next_back()
.expect("the batch parent must record a SUCCEED payload");
assert!(
parent_success_payload.contains("CUSTOM_COMPLETION_SUCCEEDED"),
"recorded payload must carry the predicate reason, got: {parent_success_payload}"
);
}
#[tokio::test]
async fn completion_predicate_observes_settled_outcomes() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(vec![0_u32, 1, 2, 3], |_child, item, _idx| async move {
if item == 1 {
Err("item 1 fails".into())
} else {
Ok(item)
}
})
.max_concurrency(1)
.completion(
crate::builders::map_parallel::CompletionConfig::builder()
.completion_predicate(|stats| {
succeed_when(item_is(stats, 1, BatchItemStatus::Failed))
})
.build()
.expect("valid completion config"),
)
.await_batch()
.await
.expect("a predicate-completed batch must not become an operation error");
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
assert_eq!(
batch.items.len(),
2,
"items 2 and 3 must never start once the predicate fires"
);
assert_eq!(batch.failure_count(), 1);
}
#[tokio::test]
async fn empty_batch_predicate_succeeded_records_custom_succeeded() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(
Vec::<u32>::new(),
|_child, item, _idx| async move { Ok(item) },
)
.completion(
crate::builders::map_parallel::CompletionConfig::with_completion_predicate(|_| {
CompletionDecision::complete(CompletionOutcome::Succeeded)
}),
)
.await_batch()
.await
.expect("an empty batch completes immediately");
assert_eq!(batch.reason, CompletionReason::CustomCompletionSucceeded);
assert!(batch.items.is_empty());
}
#[test]
fn custom_reasons_round_trip_wire() {
for reason in [
CompletionReason::CustomCompletionSucceeded,
CompletionReason::CustomCompletionFailed,
] {
assert_eq!(CompletionReason::from_wire(reason.as_str()), reason);
assert_eq!(
CompletionReason::try_from_wire(reason.as_str()),
Some(reason)
);
}
}
#[test]
fn unknown_completion_reason_is_not_parsed() {
assert_eq!(CompletionReason::try_from_wire("NOT_A_REASON"), None);
}
#[test]
fn status_follows_custom_outcome() {
let failed_item = || {
vec![BatchItem::<u32>::new(
0,
String::new(),
BatchItemStatus::Failed,
None,
None,
None,
)]
};
let ok_item = || {
vec![BatchItem::<u32>::new(
0,
String::new(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
)]
};
assert_eq!(
BatchResult::new(failed_item(), CompletionReason::CustomCompletionSucceeded).status(),
BatchStatus::Succeeded
);
assert_eq!(
BatchResult::new(ok_item(), CompletionReason::CustomCompletionFailed).status(),
BatchStatus::Failed
);
assert_eq!(
BatchResult::new(ok_item(), CompletionReason::MinSuccessfulReached).status(),
BatchStatus::Succeeded
);
assert_eq!(
BatchResult::new(failed_item(), CompletionReason::AllCompleted).status(),
BatchStatus::Failed
);
}
#[test]
fn batch_item_new_constructs_correctly() {
let item: BatchItem<String> = BatchItem::new(
3,
"my-item".to_owned(),
BatchItemStatus::Succeeded,
Some("hello".to_owned()),
None,
None,
);
assert_eq!(item.index, 3);
assert_eq!(item.name, "my-item");
assert_eq!(item.status, BatchItemStatus::Succeeded);
assert_eq!(item.result.as_deref(), Some("hello"));
assert!(item.error_message.is_none());
assert!(item.error_type.is_none());
}
#[test]
fn batch_result_accessors_match_go_sdk() {
let result: BatchResult<i32> = BatchResult::new(
vec![
BatchItem::new(
0,
String::new(),
BatchItemStatus::Succeeded,
Some(10),
None,
None,
),
BatchItem::new(
1,
String::new(),
BatchItemStatus::Failed,
None,
Some("err".into()),
Some("Error".into()),
),
BatchItem::new(
2,
String::new(),
BatchItemStatus::Succeeded,
Some(30),
None,
None,
),
],
CompletionReason::FailureToleranceExceeded,
);
assert_eq!(result.success_count(), 2);
assert_eq!(result.failure_count(), 1);
assert_eq!(result.total_count(), 3);
assert!(result.has_failure());
assert_eq!(result.status(), BatchStatus::Failed);
let errors = result.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].index, 1);
assert_eq!(errors[0].name, "");
assert_eq!(errors[0].message, "err");
assert_eq!(errors[0].error_type, Some("Error"));
assert_eq!(result.results().collect::<Vec<_>>(), vec![&10, &30]);
assert_eq!(result.reason, CompletionReason::FailureToleranceExceeded);
}
#[test]
fn batch_result_all_succeeded() {
let result: BatchResult<&str> = BatchResult::new(
vec![
BatchItem::new(
0,
String::new(),
BatchItemStatus::Succeeded,
Some("a"),
None,
None,
),
BatchItem::new(
1,
String::new(),
BatchItemStatus::Succeeded,
Some("b"),
None,
None,
),
],
CompletionReason::AllCompleted,
);
assert!(!result.has_failure());
assert_eq!(result.status(), BatchStatus::Succeeded);
assert_eq!(result.success_count(), 2);
assert_eq!(result.failure_count(), 0);
assert_eq!(result.errors().next(), None);
}
#[test]
fn batch_status_display_preserves_wire_strings() {
assert_eq!(BatchStatus::Succeeded.as_str(), "SUCCEEDED");
assert_eq!(BatchStatus::Failed.as_str(), "FAILED");
assert_eq!(BatchStatus::Succeeded.to_string(), "SUCCEEDED");
assert_eq!(BatchStatus::Failed.to_string(), "FAILED");
}
#[test]
fn completion_reason_round_trips_through_wire() {
for reason in [
CompletionReason::AllCompleted,
CompletionReason::MinSuccessfulReached,
CompletionReason::FailureToleranceExceeded,
CompletionReason::CustomCompletionSucceeded,
CompletionReason::CustomCompletionFailed,
] {
assert_eq!(CompletionReason::from_wire(reason.as_str()), reason);
}
assert_eq!(
CompletionReason::from_wire("SOME_FUTURE_REASON"),
CompletionReason::AllCompleted
);
}
#[test]
fn errors_carry_item_index_name_and_message() {
let result: BatchResult<i32> = BatchResult::new(
vec![
BatchItem::new(
0,
"a".to_owned(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
),
BatchItem::new(
1,
"b".to_owned(),
BatchItemStatus::Failed,
None,
Some("first failure".into()),
Some("Error".into()),
),
BatchItem::new(
2,
"c".to_owned(),
BatchItemStatus::Succeeded,
Some(3),
None,
None,
),
BatchItem::new(
4,
"e".to_owned(),
BatchItemStatus::Failed,
None,
Some("second failure".into()),
Some("TimeoutError".into()),
),
],
CompletionReason::AllCompleted,
);
let errors = result.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 2);
assert_eq!(errors[0].index, 1);
assert_eq!(errors[0].name, "b");
assert_eq!(errors[0].message, "first failure");
assert_eq!(errors[0].error_type, Some("Error"));
assert_eq!(errors[1].index, 4);
assert_eq!(errors[1].name, "e");
assert_eq!(errors[1].message, "second failure");
assert_eq!(errors[1].error_type, Some("TimeoutError"));
}
#[test]
fn errors_include_failed_items_without_messages() {
let result: BatchResult<i32> = BatchResult::new(
vec![BatchItem::new(
0,
"x".to_owned(),
BatchItemStatus::Failed,
None,
None,
None,
)],
CompletionReason::AllCompleted,
);
let errors = result.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].index, 0);
assert_eq!(errors[0].name, "x");
assert_eq!(errors[0].message, "");
assert_eq!(errors[0].error_type, None);
}
#[tokio::test]
async fn batch_checkpoint_round_trips_error_type_and_message() {
let mut failed_item = BatchItem::new(
1,
"boom".to_owned(),
BatchItemStatus::Failed,
None,
Some("it broke".to_owned()),
Some("CustomFailure".to_owned()),
);
failed_item.error_data = Some(r#"{"attempt":3}"#.to_owned());
failed_item.stack_trace = vec!["frame-a".to_owned(), "frame-b".to_owned()];
let original: BatchResult<i32> = BatchResult::new(
vec![
BatchItem::new(
0,
"ok".to_owned(),
BatchItemStatus::Succeeded,
Some(7),
None,
None,
),
failed_item,
],
CompletionReason::FailureToleranceExceeded,
);
let json = Arc::new(crate::serdes::JsonSerdes);
let payload = from_batch_result(
&original,
&[Some("7".to_owned()), None],
&ItemEncoding::AllJson,
);
let replayed: BatchResult<i32> = to_batch_result(
&test_ctx(CheckpointLog::empty()),
&payload,
&json,
ItemEncoding::AllJson,
2,
"arn:test",
&ItemIds::new(&OperationId::new("1".to_owned()), 2),
)
.await
.expect("deserializing the payload must succeed");
assert_eq!(replayed.reason, CompletionReason::FailureToleranceExceeded);
assert_eq!(replayed.items.len(), 2);
let failed = &replayed.items[1];
assert_eq!(failed.index, 1);
assert_eq!(failed.name, "boom");
assert_eq!(failed.status, BatchItemStatus::Failed);
assert_eq!(failed.error_message.as_deref(), Some("it broke"));
assert_eq!(failed.error_type.as_deref(), Some("CustomFailure"));
assert_eq!(failed.error_data.as_deref(), Some(r#"{"attempt":3}"#));
assert_eq!(
failed.stack_trace,
["frame-a".to_owned(), "frame-b".to_owned()]
);
let errors = replayed.errors().collect::<Vec<_>>();
assert_eq!(errors[0].error_type, Some("CustomFailure"));
assert_eq!(errors[0].message, "it broke");
let ok = &replayed.items[0];
assert_eq!(ok.result, Some(7));
assert!(ok.error_type.is_none());
assert!(ok.error_data.is_none());
assert_eq!(ok.stack_trace, Vec::<String>::new());
}
#[test]
fn from_batch_result_defaults_missing_error_type() {
let original: BatchResult<i32> = BatchResult::new(
vec![BatchItem::new(
0,
String::new(),
BatchItemStatus::Failed,
None,
Some("boom".to_owned()),
None,
)],
CompletionReason::AllCompleted,
);
let payload = record_json(&from_batch_result(
&original,
&[None],
&ItemEncoding::AllJson,
));
assert_eq!(
payload.pointer("/all/0/error/Cause/ErrorType"),
Some(&serde_json::json!("Error"))
);
assert_eq!(
payload.pointer("/all/0/error/ErrorMessage"),
Some(&serde_json::json!("boom"))
);
let untyped: BatchResult<i32> = BatchResult::new(
vec![BatchItem::new(
0,
String::new(),
BatchItemStatus::Failed,
None,
None,
None,
)],
CompletionReason::AllCompleted,
);
let payload = record_json(&from_batch_result(
&untyped,
&[None],
&ItemEncoding::AllJson,
));
assert_eq!(
payload.pointer("/all/0/error/Cause/ErrorMessage"),
Some(&serde_json::json!(""))
);
}
#[tokio::test]
async fn to_batch_result_missing_err_type_is_none() {
let payload = r#"{"all":[{"index":0,"name":"map-item-0","status":"FAILED","error":{"ErrorMessage":"boom"}}],"completionReason":"ALL_COMPLETED"}"#;
let json = Arc::new(crate::serdes::JsonSerdes);
let replayed: BatchResult<i32> = to_batch_result(
&test_ctx(CheckpointLog::empty()),
payload,
&json,
ItemEncoding::AllJson,
2,
"arn:test",
&ItemIds::new(&OperationId::new("1".to_owned()), 2),
)
.await
.expect("deserializing the payload must succeed");
let item = replayed.items.first().expect("one item");
assert_eq!(item.error_message.as_deref(), Some("boom"));
assert_eq!(item.error_type, None);
assert_eq!(item.error_data, None);
assert_eq!(item.stack_trace, Vec::<String>::new());
assert_eq!(replayed.errors().next().map(|e| e.error_type), Some(None));
}
#[tokio::test]
async fn replay_frozen_batch_preserves_error_type() {
let payload = serde_json::json!({
"all": [
{"index": 0, "name": "map-item-0", "status": "SUCCEEDED", "result": "hello"},
{
"index": 1,
"name": "map-item-1",
"status": "FAILED",
"error": {
"ErrorType": "ChildContextError",
"Cause": {"ErrorType": "ChildFnError", "ErrorMessage": "boom"}
}
}
],
"completionReason": "ALL_COMPLETED"
});
let log = CheckpointLog::from_records(vec![{
let wire_id = crate::engine::compute_wire_id_public("1");
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some(payload.to_string()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}]);
let ctx = test_ctx(log);
let batch = ctx
.map(
vec!["a".to_owned(), "b".to_owned()],
|_child, _item: String, _idx| async move {
Err::<String, _>("should not execute during replay".into())
},
)
.await_batch()
.await
.expect("frozen batch must replay from the payload");
assert_eq!(batch.success_count(), 1);
assert_eq!(batch.failure_count(), 1);
let errors = batch.errors().collect::<Vec<_>>();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].index, 1);
assert_eq!(errors[0].message, "boom");
assert_eq!(errors[0].error_type, Some("ChildFnError"));
}
#[test]
fn nesting_mode_default_is_normal() {
assert_eq!(NestingMode::default(), NestingMode::Normal);
}
#[test]
fn nesting_mode_equality() {
assert_ne!(NestingMode::Flat, NestingMode::Normal);
}
#[test]
fn completion_reason_as_str_values() {
assert_eq!(CompletionReason::AllCompleted.as_str(), "ALL_COMPLETED");
assert_eq!(
CompletionReason::MinSuccessfulReached.as_str(),
"MIN_SUCCESSFUL_REACHED"
);
assert_eq!(
CompletionReason::FailureToleranceExceeded.as_str(),
"FAILURE_TOLERANCE_EXCEEDED"
);
assert_eq!(
CompletionReason::CustomCompletionSucceeded.as_str(),
"CUSTOM_COMPLETION_SUCCEEDED"
);
assert_eq!(
CompletionReason::CustomCompletionFailed.as_str(),
"CUSTOM_COMPLETION_FAILED"
);
}
#[tokio::test]
async fn early_stop_recheck_after_semaphore_acquire() {
let sem = Arc::new(Semaphore::new(1));
let stopped = Arc::new(AtomicBool::new(false));
let launched = Arc::new(AtomicUsize::new(0));
let mut handles = Vec::new();
for i in 0..3 {
let sem = Arc::clone(&sem);
let stopped = Arc::clone(&stopped);
let launched = Arc::clone(&launched);
handles.push(tokio::spawn(async move {
let permit = sem.acquire_owned().await.unwrap();
if stopped.load(Ordering::SeqCst) {
drop(permit);
return;
}
launched.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
if i == 0 {
stopped.store(true, Ordering::SeqCst);
}
drop(permit);
}));
}
for h in handles {
h.await.unwrap();
}
assert_eq!(
launched.load(Ordering::SeqCst),
1,
"only one item should launch before stopped is honoured"
);
}
#[test]
fn flat_child_context_reports_batch_parent_wire_id() {
let ctx = test_ctx(CheckpointLog::empty());
let batch_op = ctx.mint_id();
let batch_wire = batch_op.wire().to_owned();
let child_op = ctx.mint_id();
let child_positional = child_op.positional().to_owned();
let flat_child = ctx.new_flat_child(&child_positional, &batch_wire, None);
assert_eq!(
flat_child.parent_wire_id(),
Some(batch_wire.as_str()),
"flat child must report the batch parent's wire ID"
);
}
fn item(index: usize, status: BatchItemStatus, result: Option<i32>) -> BatchItem<i32> {
BatchItem::new(index, format!("item-{index}"), status, result, None, None)
}
fn failed_item(index: usize, message: &str, error_type: Option<&str>) -> BatchItem<i32> {
BatchItem::new(
index,
format!("item-{index}"),
BatchItemStatus::Failed,
None,
Some(message.to_owned()),
error_type.map(str::to_owned),
)
}
fn indexes<'a>(items: impl Iterator<Item = &'a BatchItem<i32>>) -> Vec<usize> {
items.map(|item| item.index).collect()
}
fn one_branch_mixed_serdes() -> (MixedItemSerdes, crate::BranchHandle<u32>) {
let mut set = crate::BranchSet::new();
let handle = set.add("only", |_ctx| async { Ok(1_u32) });
let (_bodies, decoders, _json) = set.into_parts();
let batch = OperationId::new("1".to_owned());
let item_ids = ItemIds::new(&batch, decoders.len());
(MixedItemSerdes::new(&item_ids, decoders), handle)
}
#[tokio::test]
async fn mixed_item_serdes_rejects_unknown_operation_id() {
let (serdes, _handle) = one_branch_mixed_serdes();
let err = serdes
.deserialize("1".to_owned(), SerdesContext::new("other", "arn:test"))
.await
.expect_err("an unknown operation ID is an error");
assert_eq!(
err.to_string(),
r#"parallel_mixed: no branch has operation ID "other""#
);
}
#[tokio::test]
async fn batch_value_reads_each_success_by_its_key() {
use std::cell::Cell;
fn assert_send_debug<T: Send + std::fmt::Debug>() {}
assert_send_debug::<BatchResult<crate::BranchValue>>();
let mut set = crate::BranchSet::new();
let label = set.add("label", |_ctx| async { Ok("ready-value".to_owned()) });
let count = set.add("count", |_ctx| async { Ok(Cell::new(4242_u32)) });
let failed = set.add("failed", |_ctx| async { Ok(0_u32) });
let started = set.add("started", |_ctx| async { Ok(0_u32) });
let never = set.add("never", |_ctx| async { Ok(0_u32) });
let mut other = crate::BranchSet::new();
let other_label = other.add("label", |_ctx| async { Ok("other".to_owned()) });
let (bodies, decoders, _json) = set.into_parts();
let decode = |index: usize, wire: &str| {
let decoder = Arc::clone(decoders.get(index).expect("the set has the branch"));
let wire = wire.to_owned();
async move {
decoder
.decode(wire, SerdesContext::new("item", "arn:test"))
.await
.expect("the wire is valid JSON of the branch's type")
}
};
let label_value = decode(0, r#""ready-value""#).await;
let (_, count_body) = bodies.into_iter().nth(1).expect("the set has branch 1");
let count_value = count_body
.start(test_ctx(CheckpointLog::empty()))
.await
.expect("the body succeeds");
let succeeded = |index: usize, name: &str, value| {
BatchItem::new(
index,
name.to_owned(),
BatchItemStatus::Succeeded,
Some(value),
None,
None,
)
};
let batch = BatchResult::new(
vec![
succeeded(0, "label", label_value),
succeeded(1, "count", count_value),
BatchItem::new(
2,
"failed".to_owned(),
BatchItemStatus::Failed,
None,
Some("boom".to_owned()),
Some("Error".to_owned()),
),
BatchItem::new(
3,
"started".to_owned(),
BatchItemStatus::Started,
None,
None,
None,
),
],
CompletionReason::AllCompleted,
);
assert_eq!(
batch.value(&label.key()).map(String::as_str),
Some("ready-value")
);
assert_eq!(batch.value(&count.key()).map(Cell::get), Some(4242));
assert_eq!(batch.value(&failed.key()), None);
assert_eq!(batch.value(&started.key()), None);
assert_eq!(batch.value(&never.key()), None);
assert_eq!(batch.value(&other_label.key()), None);
let first = batch.items.first().expect("the batch has item 0");
let first = format!("{first:?}");
assert!(
first.contains("result: Some(BranchValue { .. })"),
"{first}"
);
assert!(!first.contains("ready-value"), "{first}");
assert_eq!(
format!("{:?}", batch.results().collect::<Vec<_>>()),
"[BranchValue { .. }, BranchValue { .. }]"
);
let text = format!("{batch:?}");
assert!(text.contains("BranchValue { .. }"), "{text}");
assert!(!text.contains("ready-value"), "{text}");
assert!(!text.contains("4242"), "{text}");
let misplaced = BatchResult::new(
vec![succeeded(0, "label", decode(1, "4242").await)],
CompletionReason::AllCompleted,
);
assert_eq!(misplaced.value(&label.key()), None);
}
#[tokio::test]
async fn mixed_item_serdes_uses_item_id_table() {
let mut set = crate::BranchSet::new();
let _first = set.add("first", |_ctx| async { Ok(1_u32) });
let second = set.add("second", |_ctx| async { Ok(2_u32) });
let (_bodies, decoders, _json) = set.into_parts();
let batch = OperationId::new("1".to_owned());
let item_ids = ItemIds::new(&batch, decoders.len());
let serdes = MixedItemSerdes::new(&item_ids, decoders);
let child = SerdesContext::new(crate::engine::compute_wire_id_public("1-2"), "arn:test");
let value = serdes
.deserialize("7".to_owned(), child)
.await
.expect("the ID 1-2 names branch 1");
value.publish();
assert_eq!(second.into_result().expect("branch 1 holds a value"), 7);
}
#[test]
fn publish_outcome_publishes_nothing_for_an_error_or_a_valueless_item() {
let mut set = crate::BranchSet::new();
let handle = set.add("only", |_ctx| async { Ok(1_u32) });
let (_bodies, decoders, _json) = set.into_parts();
let err = publish_outcome(Err(batch_error("checkpoint failed")), &decoders)
.expect_err("the batch error is returned");
assert_eq!(
format!("{err:#}"),
format!("{:#}", batch_error("checkpoint failed"))
);
let item = BatchItem::<crate::future::BranchValue>::new(
0,
"only".to_owned(),
BatchItemStatus::Succeeded,
None,
None,
None,
);
let reason = publish_outcome(
Ok(BatchResult::new(vec![item], CompletionReason::AllCompleted)),
&decoders,
)
.expect("an all-completed batch succeeds");
assert_eq!(reason, CompletionReason::AllCompleted);
let err = handle.into_result().expect_err("the handle has no result");
assert!(matches!(
err.kind(),
OperationErrorKind::ChildContext(child)
if matches!(child.kind(), ChildContextErrorKind::NotCompleted)
));
}
#[test]
fn parallel_mixed_builder_debug_shows_the_name() {
let ctx = test_ctx(CheckpointLog::empty());
let builder = ctx.parallel_mixed(crate::BranchSet::new()).name("pm");
assert_eq!(
format!("{builder:?}"),
r#"ParallelMixedBuilder { name: Some("pm"), .. }"#
);
}
#[test]
fn publish_outcome_skips_item_without_branch() {
let mut set = crate::BranchSet::new();
let handle = set.add("only", |_ctx| async { Ok(1_u32) });
let (_bodies, decoders, _json) = set.into_parts();
let failed = |index: usize| {
BatchItem::<crate::future::BranchValue>::new(
index,
format!("item-{index}"),
BatchItemStatus::Failed,
None,
Some(format!("boom-{index}")),
Some(format!("Type{index}")),
)
};
let batch = BatchResult::new(
vec![failed(0), failed(1)],
CompletionReason::FailureToleranceExceeded,
);
let err = publish_outcome(Ok(batch), &decoders)
.expect_err("a batch over its failure tolerance fails");
let wire = err.wire().expect("the error carries item 0's record");
assert_eq!(wire.error_type(), Some("Type0"));
assert_eq!(wire.error_message(), Some("boom-0"));
let held = handle
.into_result()
.expect_err("handle 0 holds item 0's error");
let held = held.wire().expect("the handle's error carries a record");
assert_eq!(held.error_type(), Some("Type0"));
assert_eq!(held.error_message(), Some("boom-0"));
}
#[test]
fn tolerance_exceeded_without_failed_item_is_an_internal_error() {
let batch = || {
BatchResult::new(
vec![item(0, BatchItemStatus::Succeeded, Some(1))],
CompletionReason::FailureToleranceExceeded,
)
};
let is_tolerance_error = |err: &OperationError| {
matches!(
err.kind(),
OperationErrorKind::ChildContext(child)
if matches!(child.kind(), ChildContextErrorKind::Internal)
) && crate::error::chain_string(err).contains("failure tolerance exceeded")
};
let err = await_error(&batch()).expect("the reason is a failing reason");
assert!(is_tolerance_error(&err), "{err:?}");
let err = collect_successful(batch()).expect_err("parallel fails the batch");
assert!(is_tolerance_error(&err), "{err:?}");
let mixed = BatchResult::<crate::future::BranchValue>::new(
Vec::new(),
CompletionReason::FailureToleranceExceeded,
);
let err = publish_outcome(Ok(mixed), &[]).expect_err("parallel_mixed fails the batch");
assert!(is_tolerance_error(&err), "{err:?}");
}
#[test]
fn batch_result_views_split_by_status() {
let batch = BatchResult::new(
vec![
item(0, BatchItemStatus::Succeeded, Some(1)),
failed_item(1, "boom", None),
item(2, BatchItemStatus::Started, None),
item(3, BatchItemStatus::Succeeded, Some(4)),
],
CompletionReason::MinSuccessfulReached,
);
assert_eq!(indexes(batch.succeeded()), [0, 3]);
assert_eq!(indexes(batch.failed()), [1]);
assert_eq!(indexes(batch.started()), [2]);
assert_eq!(batch.started_count(), 1);
}
#[test]
fn into_result_returns_first_failure() {
let batch = BatchResult::new(
vec![
item(0, BatchItemStatus::Succeeded, Some(1)),
failed_item(1, "a", Some("AError")),
failed_item(2, "b", None),
],
CompletionReason::AllCompleted,
);
let err = batch.into_result().expect_err("an item failed");
let OperationErrorKind::ChildContext(child_err) = err.kind() else {
unreachable!("expected a child-context error, got {:?}", err.kind());
};
assert!(matches!(
child_err.kind(),
ChildContextErrorKind::ChildFailed
));
let wire = err.wire().expect("the item's wire record is attached");
assert_eq!(wire.error_type(), Some("AError"));
assert_eq!(wire.error_message(), Some("a"));
}
#[test]
fn into_result_returns_completion_error_without_failed_item() {
let batch = BatchResult::new(
vec![item(0, BatchItemStatus::Succeeded, Some(1))],
CompletionReason::CustomCompletionFailed,
);
let err = batch.into_result().expect_err("the batch status is Failed");
let OperationErrorKind::ChildContext(child_err) = err.kind() else {
unreachable!("expected a child-context error, got {:?}", err.kind());
};
assert!(matches!(
child_err.kind(),
ChildContextErrorKind::BatchCompletionFailed
));
let wire = err.wire().expect("the completion wire record is attached");
assert_eq!(wire.error_type(), Some("BatchCompletionError"));
}
#[test]
fn into_result_keeps_a_successful_batch() {
let batch = BatchResult::new(
vec![
item(0, BatchItemStatus::Succeeded, Some(1)),
item(1, BatchItemStatus::Started, None),
],
CompletionReason::MinSuccessfulReached,
);
let batch = batch.into_result().expect("no item failed");
let items: Vec<_> = batch
.items
.iter()
.map(|item| (item.index, item.status, item.result))
.collect();
assert_eq!(
items,
[
(0, BatchItemStatus::Succeeded, Some(1)),
(1, BatchItemStatus::Started, None),
]
);
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
}
#[derive(Debug)]
struct TokenValidatingClient {
current_token: std::sync::Mutex<String>,
counter: std::sync::Mutex<u32>,
update_count: std::sync::Mutex<u32>,
stale_token_count: std::sync::Mutex<u32>,
}
impl TokenValidatingClient {
fn new(initial_token: &str) -> Self {
Self {
current_token: std::sync::Mutex::new(initial_token.to_owned()),
counter: std::sync::Mutex::new(0),
update_count: std::sync::Mutex::new(0),
stale_token_count: std::sync::Mutex::new(0),
}
}
fn update_count(&self) -> u32 {
*self
.update_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn stale_token_count(&self) -> u32 {
*self
.stale_token_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
impl crate::client::ExecutionClient for TokenValidatingClient {
fn checkpoint(
&self,
_execution_arn: &str,
checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::CheckpointOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
let submitted_token = checkpoint_token.to_owned();
let carried = u32::try_from(updates.len()).expect("a small request");
Box::pin(async move {
tokio::task::yield_now().await;
let mut current = self
.current_token
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if *current != submitted_token {
let mut stale = self
.stale_token_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*stale += 1;
return Err(crate::client::ExecutionClientError::non_retryable(format!(
"stale checkpoint token: expected {}, got {}",
*current, submitted_token
)));
}
let mut counter = self
.counter
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*counter += 1;
let new_token = format!("token-{counter}");
*current = new_token.clone();
let mut update_count = self
.update_count
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*update_count += carried;
Ok(crate::client::CheckpointOutput {
checkpoint_token: Some(new_token),
updated_operations: Vec::new(),
next_marker: None,
})
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::GetStateOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
Box::pin(async { Ok(crate::client::GetStateOutput::new(Vec::new(), None)) })
}
}
#[tokio::test]
async fn concurrent_checkpoint_serialization_no_token_conflict() {
let client = Arc::new(TokenValidatingClient::new("initial-token"));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::clone(&log),
client.clone() as Arc<dyn crate::client::ExecutionClient>,
"initial-token".to_owned(),
);
let num_concurrent = 10;
let mut handles = Vec::new();
for i in 0..num_concurrent {
let ctx = ctx.clone();
handles.push(tokio::spawn(async move {
let update = OperationUpdate::builder()
.id(format!("op-{i}"))
.r#type(OperationType::Context)
.sub_type("Test")
.action(OperationAction::Succeed)
.build()
.expect("valid update");
ctx.checkpoint_updates(vec![update]).await
}));
}
let mut success_count = 0_u32;
for handle in handles {
let result = handle.await.expect("task should not panic");
assert!(
result.is_ok(),
"checkpoint should succeed due to serialization: {:?}",
result.unwrap_err()
);
success_count += 1;
}
assert_eq!(success_count, num_concurrent);
assert_eq!(client.update_count(), num_concurrent);
assert_eq!(
client.stale_token_count(),
0,
"no stale token conflicts should occur with serialization"
);
}
#[tokio::test]
async fn fail_fast_then_parent_terminal_checkpoint_emitted() {
let client = Arc::new(TokenValidatingClient::new("initial-token"));
let log = Arc::new(CheckpointLog::empty());
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::clone(&log),
client.clone() as Arc<dyn crate::client::ExecutionClient>,
"initial-token".to_owned(),
);
let result: Result<Vec<i32>, _> = ctx
.parallel(vec![
crate::future::Branch::new("fail", |_ctx| {
Box::pin(async { Err("intentional".into()) })
}),
crate::future::Branch::new("ok-1", |_ctx| Box::pin(async { Ok(1) })),
crate::future::Branch::new("ok-2", |_ctx| Box::pin(async { Ok(2) })),
])
.completion(
crate::builders::map_parallel::CompletionConfig::builder()
.tolerated_failure_count(0)
.build()
.expect("valid completion config"),
)
.await;
assert!(
result.is_err(),
"parallel with fail-fast should surface the first failure"
);
assert_eq!(
client.stale_token_count(),
0,
"all checkpoints must serialize without token conflicts"
);
assert!(
client.update_count() >= 4,
"expected at least 4 checkpoint updates (parent start + child start + child fail + parent succeed), got {}",
client.update_count()
);
}
async fn panic_of<F, Fut>(run: F) -> (Option<&'static str>, bool)
where
F: FnOnce(DurableContext) -> Fut,
Fut: Future<Output = ()> + Send + 'static,
{
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let handler = run(ctx);
let outcome = crate::driver::drive_outcome(
async move {
handler.await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
ownership,
signal,
root,
None,
)
.await;
let Err(payload) = outcome else {
unreachable!("the item panic must reach the invocation driver, got {outcome:?}");
};
let wrote_fail = client
.recorded_updates()
.iter()
.any(|u| matches!(u.action(), OperationAction::Fail));
(
crate::task::test_support::str_payload(payload.as_ref()),
wrote_fail,
)
}
#[tokio::test]
async fn map_item_panic_propagates_payload() {
let (message, wrote_fail) = panic_of(|ctx| async move {
let _ = ctx
.map(vec![0_i32, 1], |_child, _item: i32, idx| async move {
assert!(idx != 1, "boom in map branch");
Ok(idx)
})
.await;
})
.await;
assert_eq!(message, Some("boom in map branch"));
assert!(!wrote_fail, "a panic records no FAIL");
}
#[tokio::test]
#[expect(clippy::panic)] async fn parallel_branch_panic_propagates_payload() {
use crate::future::Branch;
let (message, wrote_fail) = panic_of(|ctx| async move {
let _ = ctx
.parallel(vec![
Branch::new("steady", |_ctx| async move { Ok(1_i32) }),
Branch::new(
"boom",
|_ctx| async move { panic!("boom in parallel branch") },
),
])
.await;
})
.await;
assert_eq!(message, Some("boom in parallel branch"));
assert!(!wrote_fail, "a panic records no FAIL");
}
fn assert_batch_no_runtime_error<F, Fut>(log: CheckpointLog, run: F)
where
F: FnOnce(DurableContext) -> Fut + Send + 'static,
Fut: Future<Output = Result<Vec<i32>, OperationError>>,
{
let polled = crate::task::test_support::poll_once_without_runtime(move || {
run(test_ctx_with_client(log).0)
});
let std::task::Poll::Ready(Err(err)) = polled else {
unreachable!("expected an immediate error, got {polled:?}");
};
assert!(
format!("{err:#}").contains(crate::task::NO_RUNTIME_MESSAGE),
"unexpected error: {err:#}"
);
}
#[test]
fn map_without_runtime_returns_error() {
assert_batch_no_runtime_error(CheckpointLog::empty(), |ctx| {
ctx.map(
vec![0_i32],
|_child, item: i32, _idx| async move { Ok(item) },
)
.into_future()
});
}
#[test]
fn parallel_without_runtime_returns_error() {
use crate::future::Branch;
assert_batch_no_runtime_error(CheckpointLog::empty(), |ctx| {
ctx.parallel(vec![Branch::new("one", |_ctx| async move { Ok(1_i32) })])
.into_future()
});
}
#[test]
fn map_reconstruction_without_runtime_returns_error() {
let log = CheckpointLog::from_records(vec![replay_children_success_record("2")]);
assert_batch_no_runtime_error(log, |ctx| {
ctx.map(
vec![1_i32],
|_child, item: i32, _idx| async move { Ok(item) },
)
.into_future()
});
}
#[tokio::test]
#[expect(clippy::panic)] async fn map_replay_children_reconstruction_panic_unwinds_without_fail_checkpoint() {
let log = CheckpointLog::from_records(vec![replay_children_success_record("2")]);
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let task_client = Arc::clone(&client);
let handle = tokio::spawn(async move {
let ctx = DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
task_client as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
let result: Result<Vec<i32>, OperationError> = ctx
.map(vec![1_i32], |_child, _item: i32, _idx| async move {
panic!("boom in map reconstruction")
})
.await;
result
});
let join_err = handle
.await
.expect_err("a reconstruction panic must unwind the coordinator, not settle");
assert!(
join_err.is_panic(),
"the coordinator must rethrow the original panic payload"
);
let payload = join_err.into_panic();
let message = payload
.downcast_ref::<&str>()
.map(|s| (*s).to_owned())
.or_else(|| payload.downcast_ref::<String>().cloned());
assert_eq!(
message.as_deref(),
Some("boom in map reconstruction"),
"the rethrown payload must be the reconstruction panic's own"
);
assert!(
client
.recorded_updates()
.iter()
.all(|u| !matches!(u.action(), OperationAction::Fail)),
"a reconstruction panic must not checkpoint any failure"
);
}
#[tokio::test]
#[expect(clippy::panic)] async fn parallel_replay_children_reconstruction_panic_unwinds_without_fail_checkpoint() {
use crate::future::Branch;
let log = CheckpointLog::from_records(vec![
succeeded_record("2", "7"),
replay_children_success_record("3"),
]);
let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let task_client = Arc::clone(&client);
let handle = tokio::spawn(async move {
let ctx = DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
task_client as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
let result: Result<Vec<i32>, OperationError> = ctx
.parallel(vec![
Branch::new("steady", |_ctx| async move { Ok(999_i32) }),
Branch::new("boom", |_ctx| async move {
panic!("boom in parallel reconstruction")
}),
])
.await;
result
});
let join_err = handle
.await
.expect_err("a reconstruction panic must unwind the coordinator, not settle");
assert!(
join_err.is_panic(),
"the coordinator must rethrow the original panic payload"
);
let payload = join_err.into_panic();
let message = payload
.downcast_ref::<&str>()
.map(|s| (*s).to_owned())
.or_else(|| payload.downcast_ref::<String>().cloned());
assert_eq!(
message.as_deref(),
Some("boom in parallel reconstruction"),
"the rethrown payload must be the reconstruction panic's own"
);
assert!(
client
.recorded_updates()
.iter()
.all(|u| !matches!(u.action(), OperationAction::Fail)),
"a reconstruction panic must not checkpoint any failure"
);
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
struct Doc {
label: String,
nested: Vec<Vec<i64>>,
}
fn probe_doc() -> Doc {
Doc {
label: "quote:\" backslash:\\ newline:\n tab:\t ünïcodé ☃".to_owned(),
nested: vec![vec![1, -2, i64::MIN], Vec::new()],
}
}
fn child_success_payloads(client: &crate::client::InMemoryExecutionClient) -> Vec<String> {
client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Succeed) && u.parent_id().is_some())
.filter_map(|u| u.payload().map(str::to_owned))
.collect()
}
#[tokio::test]
async fn custom_serdes_is_equivalent_on_step_map_and_parallel_paths() {
use crate::future::Branch;
use crate::serdes::test_support::{HexEnvelopeSerdes, hex_envelope};
let doc = probe_doc();
let expected_wire = hex_envelope(&serde_json::to_string(&doc).expect("doc is JSON-able"));
assert!(
serde_json::from_str::<Doc>(&expected_wire).is_err(),
"the probe wire form must be unparseable by plain serde_json"
);
let (step_ctx, step_client) = test_ctx_with_client(CheckpointLog::empty());
let step_doc = doc.clone();
let step_out: Doc = step_ctx
.step(move |_| {
let d = step_doc.clone();
async move { Ok(d) }
})
.serdes(HexEnvelopeSerdes)
.await
.expect("step with a string-transform serdes must succeed");
assert_eq!(step_out, doc, "step must round-trip the value");
let step_wire: Vec<String> = step_client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Succeed))
.filter_map(|u| u.payload().map(str::to_owned))
.collect();
assert_eq!(step_wire, vec![expected_wire.clone()]);
let (map_ctx, map_client) = test_ctx_with_client(CheckpointLog::empty());
let map_doc = doc.clone();
let map_out: Vec<Doc> = map_ctx
.map(vec![0_usize], move |_child, _item, _idx| {
let d = map_doc.clone();
async move { Ok(d) }
})
.serdes(HexEnvelopeSerdes)
.await
.expect("map with a string-transform item serdes must succeed");
assert_eq!(map_out, vec![doc.clone()], "map must round-trip the value");
assert_eq!(
child_success_payloads(&map_client),
vec![expected_wire.clone()],
"a map item must produce the same wire form as a step"
);
let (par_ctx, par_client) = test_ctx_with_client(CheckpointLog::empty());
let par_doc = doc.clone();
let par_out: Vec<Doc> = par_ctx
.parallel(vec![Branch::new("only", move |_c| {
let d = par_doc.clone();
async move { Ok(d) }
})])
.serdes(HexEnvelopeSerdes)
.await
.expect("parallel with a string-transform item serdes must succeed");
assert_eq!(par_out, vec![doc], "parallel must round-trip the value");
assert_eq!(
child_success_payloads(&par_client),
vec![expected_wire],
"a parallel branch must produce the same wire form as a step"
);
}
#[tokio::test]
#[expect(clippy::too_many_lines)] async fn custom_serdes_receives_its_typed_value_on_every_path() {
use crate::future::Branch;
use crate::serdes::test_support::{RecordingSerdes, hex_envelope_of};
let value = "X".to_owned();
let expected = serde_json::json!("X");
let json_encoding = serde_json::to_string(&value).expect("a string is JSON-able");
assert_eq!(json_encoding, "\"X\"");
let wire = hex_envelope_of(&expected);
assert!(
serde_json::from_str::<String>(&wire).is_err(),
"the probe wire form must be unparseable by plain serde_json, so a \
path that skipped the transform cannot pass by accident"
);
let step_rec = RecordingSerdes::new();
let (step_ctx, step_client) = test_ctx_with_client(CheckpointLog::empty());
let step_value = value.clone();
let step_out: String = step_ctx
.step(move |_| {
let v = step_value.clone();
async move { Ok(v) }
})
.serdes(step_rec.clone())
.await
.expect("step with a recording serdes must succeed");
assert_eq!(step_out, value, "step must round-trip the value");
assert_eq!(
step_rec.serialize_inputs(),
vec![expected.clone()],
"a step serdes must be handed the value, not its JSON encoding"
);
let step_wire: Vec<String> = step_client
.recorded_updates()
.iter()
.filter(|u| matches!(u.action(), OperationAction::Succeed))
.filter_map(|u| u.payload().map(str::to_owned))
.collect();
assert_eq!(step_wire, vec![wire.clone()]);
let invoke_rec = RecordingSerdes::new();
let (invoke_ctx, _invoke_client) = test_ctx_with_client(CheckpointLog::empty());
let signal = Arc::clone(invoke_ctx.coordinator());
let invoke_fut = invoke_ctx
.invoke::<String>("target-fn", value.clone())
.payload_serdes(invoke_rec.clone());
let outcome = crate::driver::test_support::outcome_of(
invoke_ctx.clone(),
Arc::clone(invoke_ctx.task_ownership()),
signal,
invoke_fut,
)
.await;
assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
assert_eq!(
invoke_rec.serialize_inputs(),
vec![expected.clone()],
"an invoke payload serdes must be handed the same value shape as a step"
);
let callback_rec = RecordingSerdes::new();
let callback_ctx = SerdesContext::new("cb-1", "arn:test");
let callback_out: String = crate::callback::deserialize_callback_result(
&step_ctx,
None,
&callback_rec.clone(),
wire.clone(),
callback_ctx,
)
.await
.expect("the callback path must decode the wire form the step path wrote");
assert_eq!(callback_out, value, "callback must round-trip the value");
assert_eq!(
callback_rec.deserialize_outputs(),
vec![expected.clone()],
"the callback path must reconstruct the same value shape the step \
path handed the serdes"
);
let map_rec = RecordingSerdes::new();
let (map_ctx, _map_client) = test_ctx_with_client(CheckpointLog::empty());
let map_value = value.clone();
let map_out: Vec<String> = map_ctx
.map(vec![0_usize], move |_child, _item, _idx| {
let v = map_value.clone();
async move { Ok(v) }
})
.serdes(map_rec.clone())
.await
.expect("map with a recording item serdes must succeed");
assert_eq!(map_out, vec![value.clone()]);
let map_inputs = map_rec.serialize_inputs();
assert_eq!(
map_inputs.len(),
1,
"the item serdes serializes the item once, for the child \
checkpoint, got {map_inputs:?}"
);
assert_eq!(
map_rec.distinct_serialize_inputs(),
vec![expected.clone()],
"every map item serdes call must be handed the identical value \
shape, got {map_inputs:?}"
);
let par_rec = RecordingSerdes::new();
let (par_ctx, _par_client) = test_ctx_with_client(CheckpointLog::empty());
let par_value = value.clone();
let par_out: Vec<String> = par_ctx
.parallel(vec![Branch::new("only", move |_c| {
let v = par_value.clone();
async move { Ok(v) }
})])
.serdes(par_rec.clone())
.await
.expect("parallel with a recording item serdes must succeed");
assert_eq!(par_out, vec![value.clone()]);
assert_eq!(
par_rec.distinct_serialize_inputs(),
vec![expected.clone()],
"a parallel branch serdes must be handed the same value shape"
);
let batch_rec = RecordingSerdes::new();
let (batch_ctx, _batch_client) = test_ctx_with_client(CheckpointLog::empty());
let batch_value = value.clone();
let batch_out: Vec<String> = batch_ctx
.map(vec![0_usize], move |_child, _item, _idx| {
let v = batch_value.clone();
async move { Ok(v) }
})
.result_serdes(batch_rec.clone())
.await
.expect("map with a recording result serdes must succeed");
assert_eq!(batch_out, vec![value]);
let batch_inputs = batch_rec.serialize_inputs();
assert_eq!(
batch_inputs,
vec![serde_json::Value::String(format!(
r#"{{"all":[{{"index":0,"status":"SUCCEEDED","name":"map-item-0","result":{json_encoding}}}],"completionReason":"ALL_COMPLETED"}}"#
))],
"the batch result serdes must be handed the record text once"
);
}
#[tokio::test]
async fn custom_serdes_item_replay_reverses_the_transform() {
use crate::serdes::test_support::{HexEnvelopeSerdes, hex_envelope};
let doc = probe_doc();
let wire = hex_envelope(&serde_json::to_string(&doc).expect("doc is JSON-able"));
let parent_wire_id = crate::engine::compute_wire_id_public("1");
let child_wire_id = crate::engine::compute_wire_id_public("1-1");
let log = CheckpointLog::from_records(vec![
(
parent_wire_id.clone(),
CheckpointRecord {
id: parent_wire_id,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
(
child_wire_id.clone(),
CheckpointRecord {
id: child_wire_id,
status: CheckpointStatus::Succeeded,
result: Some(wire),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
),
]);
let (ctx, _client) = test_ctx_with_client(log);
let out: Vec<Doc> = ctx
.map(vec![0_usize], |_child, _item, _idx| async move {
unreachable!("a terminal child must be replayed, not re-executed")
})
.serdes(HexEnvelopeSerdes)
.await
.expect("replay of a custom-serdes item must succeed");
assert_eq!(out, vec![probe_doc()]);
}
#[tokio::test]
async fn filesystem_serdes_works_as_a_map_item_serdes() {
let tmp = std::env::temp_dir().join("map_item_fs_serdes");
let _ = std::fs::remove_dir_all(&tmp);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let out: Vec<String> = ctx
.map(vec![1_i32, 2, 3], |_child, item, _idx| async move {
Ok(format!("item-{item}"))
})
.serdes(crate::serdes::FileSystemSerdes::new(
tmp.to_string_lossy().to_string(),
))
.await
.expect("FileSystemSerdes must work as a map item serdes");
assert_eq!(out, vec!["item-1", "item-2", "item-3"]);
let payloads = child_success_payloads(&client);
assert_eq!(payloads.len(), 3, "payloads: {payloads:?}");
let mut files: Vec<String> = payloads
.iter()
.map(|p| {
let v: serde_json::Value =
serde_json::from_str(p).expect("envelope must be valid JSON");
v.get("file")
.and_then(serde_json::Value::as_str)
.expect("envelope must be a file pointer")
.to_owned()
})
.collect();
files.sort();
files.dedup();
assert_eq!(files.len(), 3, "each item needs its own file: {files:?}");
let _ = std::fs::remove_dir_all(&tmp);
}
fn child_starts_before_first_terminal(
client: &crate::client::InMemoryExecutionClient,
) -> (usize, usize) {
let updates = client.recorded_updates();
let mut starts_before_first_terminal = 0;
let mut total_starts = 0;
let mut saw_terminal = false;
for u in &updates {
if u.parent_id().is_none() {
continue;
}
match u.action() {
OperationAction::Start => {
total_starts += 1;
if !saw_terminal {
starts_before_first_terminal += 1;
}
}
_ => saw_terminal = true,
}
}
(starts_before_first_terminal, total_starts)
}
#[tokio::test]
async fn batch_larger_than_concurrency_issues_no_start_before_first_item_runs() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let result = ctx
.map(
vec![1_i32, 2, 3, 4, 5, 6],
|_child, item, _idx| async move { Ok(item * 10) },
)
.max_concurrency(2)
.await
.expect("map must succeed");
assert_eq!(result, vec![10, 20, 30, 40, 50, 60]);
let (starts_before, total_starts) = child_starts_before_first_terminal(&client);
assert_eq!(
total_starts, 6,
"every admitted item gets exactly one Start"
);
assert!(
starts_before <= 2,
"at most `concurrency` child Starts may precede the first item \
outcome (lazy admission), found {starts_before}"
);
}
fn parent_succeed_update(client: &crate::client::InMemoryExecutionClient) -> OperationUpdate {
client
.recorded_updates()
.into_iter()
.find(|u| u.parent_id().is_none() && matches!(u.action(), OperationAction::Succeed))
.expect("batch parent SUCCEED update must be recorded")
}
#[tokio::test]
async fn large_batch_result_checkpoints_decision_record() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let big = "x".repeat(200 * 1024);
let result = ctx
.map(
vec![big.clone(), big],
|_child, item: String, _idx| async move { Ok(item) },
)
.await
.expect("map must succeed");
assert_eq!(result.len(), 2);
let parent = parent_succeed_update(&client);
assert_eq!(
parent.context_options().and_then(|c| c.replay_children),
Some(true),
"oversized batch payload must set replay_children"
);
let summary: serde_json::Value = serde_json::from_str(
parent
.payload()
.expect("large-result parent must carry the summary"),
)
.expect("payload must parse as a plain-JSON summary");
assert_eq!(summary["totalCount"], 2);
assert_eq!(summary["completionReason"], "ALL_COMPLETED");
assert_eq!(summary["itemStatuses"], "SS");
}
#[tokio::test]
async fn decision_record_total_count_is_ever_started_not_input_length() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = ctx
.map(
vec![0_usize, 1, 2, 3, 4, 5],
|_child, _item, idx| async move { Ok("y".repeat(300 * 1024) + &idx.to_string()) },
)
.max_concurrency(1)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch()
.await
.expect("map must succeed");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(batch.items.len(), 1);
let parent = parent_succeed_update(&client);
let summary: serde_json::Value = serde_json::from_str(
parent
.payload()
.expect("large-result parent must carry the summary"),
)
.expect("payload must parse as a plain-JSON summary");
assert_eq!(
summary["totalCount"], 1,
"totalCount counts items ever started, not the 6 input items"
);
assert_eq!(summary["completionReason"], "MIN_SUCCESSFUL_REACHED");
assert_eq!(summary["itemStatuses"], "S");
}
#[tokio::test]
async fn oversized_summary_bypasses_result_serdes() {
use crate::serdes::test_support::HexEnvelopeSerdes;
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let big = "z".repeat(200 * 1024);
let result = ctx
.map(
vec![big.clone(), big],
|_child, item: String, _idx| async move { Ok(item) },
)
.result_serdes(HexEnvelopeSerdes)
.await
.expect("map must succeed");
assert_eq!(result.len(), 2);
let parent = parent_succeed_update(&client);
let record: BatchDecisionRecord = serde_json::from_str(
parent
.payload()
.expect("large-result parent must carry the decision record"),
)
.expect("the decision record must parse as plain JSON, not the hex wire form");
assert_eq!(record.total_count, 2);
}
fn decision_record_parent(positional_id: &str, decision: &str) -> (String, CheckpointRecord) {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some(decision.to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: true,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
}
#[tokio::test]
async fn replay_of_threshold_stopped_large_batch_obeys_decision_record() {
let decision =
r#"{"totalCount":2,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[]}"#;
let log = CheckpointLog::from_records(vec![
decision_record_parent("1", decision),
succeeded_record("1-1", "\"hello\""),
succeeded_record("1-2", "\"world\""),
]);
let ctx = test_ctx(log);
let batch: BatchResult<String> = ctx
.map(
vec![0_usize, 1, 2, 3, 4, 5],
|_child, _item, _idx| async move {
unreachable!("replay must not re-run any item body")
},
)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(2))
.await_batch()
.await
.expect("replay must succeed");
assert_eq!(
batch.reason,
CompletionReason::MinSuccessfulReached,
"the recorded reason is taken verbatim"
);
assert_eq!(batch.items.len(), 2, "unstarted items stay omitted");
assert_eq!(batch.items[0].index, 0);
assert_eq!(batch.items[0].result.as_deref(), Some("hello"));
assert_eq!(batch.items[1].index, 1);
assert_eq!(batch.items[1].result.as_deref(), Some("world"));
}
#[tokio::test]
async fn replay_reports_started_set_as_started_without_reading_child_checkpoints() {
let started_record = |positional_id: &str| {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
};
let decision =
r#"{"totalCount":3,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[1]}"#;
let log = CheckpointLog::from_records(vec![
decision_record_parent("1", decision),
succeeded_record("1-1", "\"a\""),
started_record("1-2"),
succeeded_record("1-3", "\"c\""),
]);
let ctx = test_ctx(log);
let batch: BatchResult<String> = ctx
.map(vec![0_usize, 1, 2, 3], |_child, _item, _idx| async move {
unreachable!("replay must not re-run any item body")
})
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(2))
.await_batch()
.await
.expect("replay must succeed");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
let statuses: Vec<(usize, BatchItemStatus)> =
batch.items.iter().map(|i| (i.index, i.status)).collect();
assert_eq!(
statuses,
vec![
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started),
(2, BatchItemStatus::Succeeded),
],
"started-set item is reported STARTED; the never-started item is omitted"
);
assert!(
batch.items[1].result.is_none() && batch.items[1].error_message.is_none(),
"a STARTED item carries no result and no error"
);
assert_eq!(batch.results().collect::<Vec<_>>(), vec!["a", "c"]);
}
#[tokio::test]
async fn plan_constrained_rebuild_reads_only_admitted_child_checkpoints() {
let poisoned_record = |positional_id: &str| {
let wire_id = crate::engine::compute_wire_id_public(positional_id);
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Started,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: Some("Step".to_owned()),
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
};
let decision =
r#"{"totalCount":3,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[1]}"#;
let log = CheckpointLog::from_records(vec![
decision_record_parent("1", decision),
replay_children_success_record("1-1"),
poisoned_record("1-2"),
succeeded_record("1-3", "\"c\""),
poisoned_record("1-4"),
]);
let (ctx, _client) = test_ctx_with_client(log);
let batch: BatchResult<String> = ctx
.map(vec![0_usize, 1, 2, 3], |_child, _item, idx| async move {
assert_eq!(idx, 0, "only the replay_children item re-executes");
Ok("a".to_owned())
})
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(2))
.await_batch()
.await
.expect("the plan-constrained rebuild must not read non-admitted child checkpoints");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
let statuses: Vec<(usize, BatchItemStatus)> =
batch.items.iter().map(|i| (i.index, i.status)).collect();
assert_eq!(
statuses,
vec![
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started),
(2, BatchItemStatus::Succeeded),
],
"started-set item reported STARTED from the record; never-started omitted"
);
assert_eq!(batch.results().collect::<Vec<_>>(), vec!["a", "c"]);
}
#[tokio::test]
async fn empty_replay_children_payload_fails_replay() {
for payload in [None, Some(" ")] {
let (parent_wire, mut parent) = decision_record_parent("1", "");
parent.result = payload.map(str::to_owned);
let log = CheckpointLog::from_records(vec![
(parent_wire, parent),
succeeded_record("1-1", "\"hello\""),
succeeded_record("1-2", "\"world\""),
]);
let (ctx, _client) = test_ctx_with_client(log);
let ran = Arc::new(AtomicBool::new(false));
let ran_probe = Arc::clone(&ran);
let err = ctx
.map(
vec!["a".to_owned(), "b".to_owned()],
move |_child, item: String, _idx| {
let ran = Arc::clone(&ran_probe);
async move {
ran.store(true, Ordering::SeqCst);
Ok(item)
}
},
)
.await
.expect_err("a batch parent without its summary fails replay");
assert!(
format!("{err:#}").contains("malformed batch decision record"),
"{payload:?}: unexpected error: {err:#}"
);
assert!(
!ran.load(Ordering::SeqCst),
"{payload:?}: no item body may run"
);
}
}
#[tokio::test]
async fn flat_batch_replay_reports_started_set_from_record() {
let decision =
r#"{"totalCount":3,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[1]}"#;
let log = CheckpointLog::from_records(vec![decision_record_parent("1", decision)]);
let ctx = test_ctx(log);
let invoked = Arc::new(std::sync::Mutex::new(Vec::<usize>::new()));
let invoked_probe = Arc::clone(&invoked);
let batch = ctx
.map(vec![0_usize, 1, 2, 3, 4, 5], move |_child, _item, idx| {
let invoked = Arc::clone(&invoked_probe);
async move {
invoked
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(idx);
Ok(format!("flat-{idx}"))
}
})
.nesting(NestingMode::Flat)
.max_concurrency(2)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(2))
.await_batch()
.await
.expect("flat replay must succeed");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
let statuses: Vec<(usize, BatchItemStatus)> =
batch.items.iter().map(|i| (i.index, i.status)).collect();
assert_eq!(
statuses,
vec![
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started),
(2, BatchItemStatus::Succeeded),
],
"the started set is reported STARTED from the record; unstarted items are omitted"
);
let mut ran = invoked
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
ran.sort_unstable();
assert_eq!(
ran,
vec![0, 2],
"flat replay re-runs the completed set only — never started-set or unstarted items"
);
}
#[tokio::test]
async fn replay_plan_trigger_keeps_recorded_completed_items() {
let decision =
r#"{"totalCount":2,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[]}"#;
let log = CheckpointLog::from_records(vec![decision_record_parent("1", decision)]);
let ctx = test_ctx(log);
let finished = Arc::new(AtomicBool::new(false));
let finished_probe = Arc::clone(&finished);
let batch = tokio::time::timeout(
std::time::Duration::from_secs(5),
ctx.map(vec![0_usize, 1], move |_child, _item, idx| {
let finished = Arc::clone(&finished_probe);
async move {
if idx == 1 {
for _ in 0..20 {
tokio::task::yield_now().await;
}
finished.store(true, Ordering::SeqCst);
}
Ok(format!("flat-{idx}"))
}
})
.nesting(NestingMode::Flat)
.max_concurrency(2)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the replay resolves")
.expect("the replay succeeds");
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
let items: Vec<(usize, BatchItemStatus, Option<String>)> = batch
.items
.iter()
.map(|i| (i.index, i.status, i.result.clone()))
.collect();
assert_eq!(
items,
vec![
(0, BatchItemStatus::Succeeded, Some("flat-0".to_owned())),
(1, BatchItemStatus::Succeeded, Some("flat-1".to_owned())),
],
"both recorded completions are reconstructed"
);
assert!(
finished.load(Ordering::SeqCst),
"item 1 ran to its end and was not aborted"
);
}
#[test]
fn replay_plan_reconstructs_either_index_form() {
let record = BatchDecisionRecord {
total_count: 4,
completion_reason: "ALL_COMPLETED".to_owned(),
started_indexes: Some(vec![1, 3]),
completed_indexes: None,
};
let plan = ReplayPlan::from_record(&record, 10).expect("well-formed record");
assert_eq!(plan.total_count, 4);
assert!(plan.admits(0) && plan.admits(2));
assert!(!plan.admits(1) && !plan.admits(3));
assert!(!plan.admits(4), "beyond totalCount is never admitted");
let record = BatchDecisionRecord {
total_count: 3,
completion_reason: "MIN_SUCCESSFUL_REACHED".to_owned(),
started_indexes: None,
completed_indexes: Some(vec![2]),
};
let plan = ReplayPlan::from_record(&record, 10).expect("well-formed record");
assert!(!plan.admits(0) && !plan.admits(1) && plan.admits(2));
assert_eq!(plan.reason, CompletionReason::MinSuccessfulReached);
}
#[test]
fn replay_plan_rejects_semantically_invalid_records() {
let base = || BatchDecisionRecord {
total_count: 3,
completion_reason: "ALL_COMPLETED".to_owned(),
started_indexes: Some(vec![1]),
completed_indexes: None,
};
let record = base();
let err = ReplayPlan::from_record(&record, 2).expect_err("totalCount > input");
assert!(
crate::error::chain_string(&err).contains("exceeds the input length"),
"unexpected error: {err}"
);
let mut record = base();
record.completed_indexes = Some(vec![0, 2]);
let err = ReplayPlan::from_record(&record, 10).expect_err("both index forms");
assert!(crate::error::chain_string(&err).contains("both startedIndexes"));
let mut record = base();
record.started_indexes = None;
let err = ReplayPlan::from_record(&record, 10).expect_err("neither index form");
assert!(crate::error::chain_string(&err).contains("neither startedIndexes"));
let mut record = base();
record.started_indexes = Some(vec![3]);
let err = ReplayPlan::from_record(&record, 10).expect_err("started index out of range");
assert!(crate::error::chain_string(&err).contains("outside [0, 3)"));
let mut record = base();
record.started_indexes = None;
record.completed_indexes = Some(vec![50]);
let err = ReplayPlan::from_record(&record, 10).expect_err("completed index out of range");
assert!(crate::error::chain_string(&err).contains("outside [0, 3)"));
let mut record = base();
record.completion_reason = "SOME_FUTURE_REASON".to_owned();
let err = ReplayPlan::from_record(&record, 10).expect_err("unknown reason");
assert!(crate::error::chain_string(&err).contains("unknown completionReason"));
}
#[test]
fn parse_decision_record_rejects_what_is_not_a_record() {
assert!(parse_decision_record("not json").is_err());
assert!(parse_decision_record(r#"{"unrelated":true}"#).is_err());
let record = parse_decision_record(
r#"{"totalCount":2,"completionReason":"ALL_COMPLETED","startedIndexes":[]}"#,
)
.expect("well-formed record must parse");
assert_eq!(record.total_count, 2);
}
#[tokio::test]
async fn malformed_decision_record_fails_replay_instead_of_reexecuting() {
let log = CheckpointLog::from_records(vec![decision_record_parent(
"1",
r#"{"totalCount":"#, )]);
let ctx = test_ctx(log);
let result: Result<Vec<String>, _> = ctx
.map(vec![0_usize, 1, 2], |_child, _item, _idx| async move {
unreachable!("a corrupt decision record must not re-execute any item")
})
.await;
let Err(err) = result else {
unreachable!("replay of a corrupt decision record must fail")
};
let display = crate::error::chain_string(&err);
assert!(
display.contains("batch record is not valid"),
"expected the corrupt-record error, got: {display}"
);
}
#[tokio::test]
async fn oversized_decision_record_refuses_checkpoint_instead_of_dropping_record() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let oversized_payload = "p".repeat(CHECKPOINT_SIZE_LIMIT_BYTES + 1);
let oversized_decision = "d".repeat(CHECKPOINT_SIZE_LIMIT_BYTES + 1);
let parent_wire = crate::engine::compute_wire_id_public("1");
let result = checkpoint_batch_success_serialized(
&ctx,
&parent_wire,
None,
MAP_SUB_TYPE,
&oversized_payload,
&oversized_decision,
None,
)
.await;
let Err(err) = result else {
unreachable!("an oversized decision record must refuse the checkpoint")
};
let display = crate::error::chain_string(&err);
assert!(
display.contains("batch summary exceeds the checkpoint size limit"),
"expected the oversized-record error, got: {display}"
);
assert!(
client
.recorded_updates()
.iter()
.all(|u| !matches!(u.action(), OperationAction::Succeed)),
"no terminal SUCCEED may be written without its decision record"
);
}
#[tokio::test]
async fn batch_rebuild_validates_completed_child_identity() {
let mismatched_child = {
let wire_id = crate::engine::compute_wire_id_public("1-1");
(
wire_id.clone(),
CheckpointRecord {
id: wire_id,
status: CheckpointStatus::Succeeded,
result: Some("\"a\"".to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: Some("Context".to_owned()),
sub_type: Some("ParallelBranch".to_owned()),
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)
};
let decision = r#"{"totalCount":2,"completionReason":"ALL_COMPLETED","startedIndexes":[]}"#;
let log = CheckpointLog::from_records(vec![
decision_record_parent("1", decision),
mismatched_child,
succeeded_record("1-2", "\"b\""),
]);
let ctx = test_ctx(log);
let result: Result<BatchResult<String>, _> = ctx
.map(vec![0_usize, 1], |_child, _item, _idx| async move {
unreachable!("identity mismatch must fail before any re-execution")
})
.await_batch()
.await;
let Err(err) = result else {
unreachable!("a mismatched child identity must fail the replay")
};
assert!(
matches!(err.kind(), OperationErrorKind::NonDeterministicExecution(_)),
"expected NonDeterministicExecution, got: {err:#}"
);
}
const EARLY_BOUND: std::time::Duration = std::time::Duration::from_secs(5);
fn item_wires() -> [String; 2] {
[
crate::engine::compute_wire_id_public("1-1"),
crate::engine::compute_wire_id_public("1-2"),
]
}
fn item_action_count(
client: &crate::client::InMemoryExecutionClient,
index: usize,
action: &OperationAction,
) -> usize {
let wire = item_wires()[index].clone();
client
.recorded_updates()
.iter()
.filter(|u| u.id == wire && u.action() == action)
.count()
}
fn statuses<O>(batch: &BatchResult<O>) -> Vec<(usize, BatchItemStatus)> {
batch.items.iter().map(|i| (i.index, i.status)).collect()
}
type ItemFut = Pin<Box<dyn Future<Output = Result<i32, BoxError>> + Send>>;
fn early_body(
fail: bool,
guard: Arc<AtomicBool>,
) -> impl Fn(DurableContext, i32, usize) -> ItemFut + Send + Sync + 'static {
let running = Arc::new(tokio::sync::Notify::new());
move |child, item, _idx| {
let guard = Arc::clone(&guard);
let running = Arc::clone(&running);
Box::pin(async move {
if item == 0 {
running.notified().await;
if fail {
return Err("boom".into());
}
return Ok(child.step(|_| async { Ok(0_i32) }).name("fast").await?);
}
let _guard = SetOnDrop(guard);
running.notify_one();
Ok(child
.step(|_| std::future::pending::<Result<i32, BoxError>>())
.name("forever")
.await?)
})
}
}
struct SetOnDrop(Arc<AtomicBool>);
impl Drop for SetOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
fn no_guard() -> Arc<AtomicBool> {
Arc::new(AtomicBool::new(false))
}
#[tokio::test]
async fn early_completion_does_not_wait_for_running_item() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(false, no_guard()))
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch does not wait for its running item")
.expect("the batch succeeds");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(batch.total_count(), 2);
assert_started_defaults(&batch.items[1], "map-item-1");
assert_eq!(item_action_count(&client, 0, &OperationAction::Succeed), 1);
assert_eq!(item_action_count(&client, 1, &OperationAction::Succeed), 0);
let [settled, running] = item_wires();
assert!(!ctx.admission().admits(&[settled]), "item 0 is closed");
assert!(!ctx.admission().admits(&[running]), "item 1 is closed");
}
#[derive(Debug)]
struct GuardAtParentWrite {
guard: Arc<AtomicBool>,
seen: Arc<AtomicBool>,
}
impl crate::client::ExecutionClient for GuardAtParentWrite {
fn checkpoint(
&self,
_arn: &str,
_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::CheckpointOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
let parent = crate::engine::compute_wire_id_public("1");
let at_parent = updates
.iter()
.any(|u| u.id == parent && u.action() == &OperationAction::Succeed);
Box::pin(async move {
if at_parent {
for _ in 0..1_000 {
if self.guard.load(Ordering::SeqCst) {
break;
}
tokio::task::yield_now().await;
}
self.seen
.store(self.guard.load(Ordering::SeqCst), Ordering::SeqCst);
}
Ok(crate::client::CheckpointOutput {
checkpoint_token: Some("token".to_owned()),
updated_operations: Vec::new(),
next_marker: None,
})
})
}
fn get_state(
&self,
_arn: &str,
_token: &str,
_marker: Option<&str>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::GetStateOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
Box::pin(async {
Ok(crate::client::GetStateOutput {
operations: Vec::new(),
next_marker: None,
})
})
}
}
#[tokio::test]
async fn early_completion_aborts_running_item_before_parent_write() {
let guard = no_guard();
let seen = no_guard();
let client = Arc::new(GuardAtParentWrite {
guard: Arc::clone(&guard),
seen: Arc::clone(&seen),
});
let ctx = DurableContext::new_root_with_client(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client as Arc<dyn crate::client::ExecutionClient>,
"token0".to_owned(),
);
tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(false, Arc::clone(&guard)))
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch does not wait for its running item")
.expect("the batch succeeds");
assert!(
seen.load(Ordering::SeqCst),
"item 1's task unwound before the batch parent's SUCCEED was written"
);
}
#[tokio::test]
async fn started_branch_keeps_its_name() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.parallel(vec![
Branch::new("fast", |_child| async { Ok(0_i32) }),
Branch::new("slow", |_child| std::future::pending()),
])
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch does not wait for its running branch")
.expect("the batch succeeds");
let started = batch
.items
.iter()
.find(|item| item.status == BatchItemStatus::Started)
.expect("the running branch is reported as Started");
assert_started_defaults(started, "slow");
assert_eq!(started.index, 1);
}
fn assert_started_defaults<O: std::fmt::Debug>(item: &BatchItem<O>, name: &str) {
assert_eq!(item.status, BatchItemStatus::Started);
assert_eq!(item.name, name);
assert!(item.result.is_none(), "no result: {item:?}");
assert_eq!(item.error_message, None);
assert_eq!(item.error_type, None);
assert_eq!(item.error_data, None);
assert!(item.stack_trace.is_empty(), "no stack trace: {item:?}");
}
async fn wait_for_start(client: &crate::client::InMemoryExecutionClient, name: &str) {
for _ in 0..1_000 {
if has_start(client, name) {
return;
}
tokio::task::yield_now().await;
}
assert!(
has_start(client, name),
"the START of {name} was never recorded"
);
}
fn has_start(client: &crate::client::InMemoryExecutionClient, name: &str) -> bool {
client
.recorded_updates()
.iter()
.any(|u| u.name() == Some(name) && u.action() == &OperationAction::Start)
}
#[tokio::test(start_paused = true)]
async fn early_completion_abandons_running_item() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let observer = Arc::clone(&client);
let start = tokio::time::Instant::now();
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.map(
vec![0_i32, 1],
move |child: DurableContext, item: i32, _idx| {
let observer = Arc::clone(&observer);
async move {
if item == 0 {
wait_for_start(&observer, "a").await;
return Ok(child.step(|_| async { Ok(0_i32) }).name("fast").await?);
}
child
.step(|_| async {
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
Ok(1_i32)
})
.name("a")
.await?;
Ok(child.step(|_| async { Ok(2_i32) }).name("b").await?)
}
},
)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch returns")
.expect("the batch succeeds");
assert!(
start.elapsed() < std::time::Duration::from_millis(800),
"the batch returned after {:?}, so it waited for item 1",
start.elapsed()
);
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
assert!(has_start(&client, "a"), "item 1 entered step a");
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
assert!(
!client
.recorded_updates()
.iter()
.any(|u| u.name() == Some("b")),
"the abandoned item started no later operation"
);
assert_eq!(item_action_count(&client, 1, &OperationAction::Succeed), 0);
}
#[tokio::test]
async fn fail_fast_does_not_wait_for_running_sibling() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(true, no_guard()))
.await_batch(),
)
.await
.expect("the batch does not wait for its running item")
.expect("await_batch reports item failures as items");
assert_eq!(
statuses(&batch),
[(0, BatchItemStatus::Failed), (1, BatchItemStatus::Started)]
);
assert_eq!(batch.reason, CompletionReason::FailureToleranceExceeded);
assert_eq!(item_action_count(&client, 0, &OperationAction::Fail), 1);
assert_eq!(item_action_count(&client, 1, &OperationAction::Fail), 0);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let err = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(true, no_guard()))
.into_future(),
)
.await
.expect("the batch does not wait for its running item")
.expect_err("fail-fast ends the batch with an error");
assert!(
matches!(
err.kind(),
OperationErrorKind::ChildContext(e) if matches!(e.kind(), ChildContextErrorKind::ChildFailed)
),
"unexpected error kind: {:?}",
err.kind()
);
assert_eq!(item_action_count(&client, 0, &OperationAction::Fail), 1);
assert_eq!(item_action_count(&client, 1, &OperationAction::Fail), 0);
}
#[tokio::test]
async fn predicate_completion_does_not_wait_for_running_item() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(false, no_guard()))
.completion(
crate::builders::map_parallel::CompletionConfig::with_completion_predicate(
|s| succeed_when(s.succeeded() >= 1),
),
)
.await_batch(),
)
.await
.expect("the batch does not wait for its running item")
.expect("the batch succeeds");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
}
#[tokio::test]
async fn abandoned_item_leaves_no_task_after_invocation() {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let guard = no_guard();
let handler = {
let ctx = ctx.clone();
let guard = Arc::clone(&guard);
async move {
ctx.map(vec![0_i32, 1], early_body(false, guard))
.completion(
crate::builders::map_parallel::CompletionConfig::with_min_successful(1),
)
.await_batch()
.await
.expect("the batch succeeds");
Ok::<_, crate::error::WireError>("done".to_owned())
}
};
let outcome = tokio::time::timeout(
EARLY_BOUND,
crate::driver::drive_outcome(handler, ownership, signal, root, None),
)
.await
.expect("the invocation does not wait for the abandoned item");
assert!(
matches!(outcome, Ok(crate::driver::InvocationOutcome::Complete(_))),
"unexpected outcome: {outcome:?}"
);
assert!(
guard.load(Ordering::SeqCst),
"the abandoned item's task unwound before the invocation returned"
);
assert_eq!(ctx.coordinator().tasks().live(), 0);
}
#[tokio::test]
async fn early_completion_await_returns_only_succeeded_values() {
use crate::future::Branch;
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let values = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], early_body(false, no_guard()))
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.into_future(),
)
.await
.expect("the map does not wait for its running item")
.expect("the map succeeds");
assert_eq!(values, [0]);
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let values = tokio::time::timeout(
EARLY_BOUND,
ctx.parallel(vec![
Branch::new("fast", |_child| async { Ok(0_i32) }),
Branch::new("slow", |_child| std::future::pending()),
])
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.into_future(),
)
.await
.expect("the parallel does not wait for its running branch")
.expect("the parallel succeeds");
assert_eq!(values, [0]);
}
#[derive(Debug, Clone, Copy)]
enum Late {
Value,
Error,
Panic,
}
#[expect(clippy::panic)] async fn queued_outcome_batch(late: Late) -> BatchResult<i32> {
let (ctx, _client) = test_ctx_with_client(CheckpointLog::empty());
let tasks = Arc::clone(ctx.coordinator().tasks());
let entered = Arc::new(AtomicUsize::new(0));
let ready = Arc::new([tokio::sync::Notify::new(), tokio::sync::Notify::new()]);
let body = {
let entered = Arc::clone(&entered);
let ready = Arc::clone(&ready);
move |_child: DurableContext, item: i32, idx: usize| {
let entered = Arc::clone(&entered);
let ready = Arc::clone(&ready);
async move {
entered.fetch_add(1, Ordering::SeqCst);
ready[idx].notified().await;
if item == 0 {
return Ok(0_i32);
}
match late {
Late::Value => Ok(1_i32),
Late::Error => Err("late failure".into()),
Late::Panic => panic!("late panic"),
}
}
}
};
let mut batch = std::pin::pin!(
ctx.map(vec![0_i32, 1], body)
.nesting(NestingMode::Flat)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch()
);
let early = std::future::poll_fn(|cx| {
if entered.load(Ordering::SeqCst) == 2 {
return std::task::Poll::Ready(None);
}
if let std::task::Poll::Ready(done) = batch.as_mut().poll(cx) {
return std::task::Poll::Ready(Some(done));
}
cx.waker().wake_by_ref();
std::task::Poll::Pending
});
let early = tokio::time::timeout(EARLY_BOUND, early)
.await
.expect("both items start");
assert!(early.is_none(), "the batch completed before its items ran");
let wait_live = |n: usize| {
let tasks = Arc::clone(&tasks);
async move {
for _ in 0..10_000 {
if tasks.live() == n {
return;
}
tokio::task::yield_now().await;
}
assert_eq!(tasks.live(), n, "the live task count never reached {n}");
}
};
ready[0].notify_one();
wait_live(1).await;
ready[1].notify_one();
wait_live(0).await;
tokio::time::timeout(EARLY_BOUND, batch)
.await
.expect("the batch completes")
.expect("the late outcome is not observed")
}
#[tokio::test]
async fn finished_unjoined_item_is_started() {
for late in [Late::Value, Late::Error, Late::Panic] {
let batch = queued_outcome_batch(late).await;
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
],
"late {late:?}"
);
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_started_defaults(&batch.items[1], "map-item-1");
}
}
fn terminal_batch_payload() -> String {
serde_json::json!({
"all": [
{"index": 0, "name": "a", "status": "SUCCEEDED", "result": "hello"},
{"index": 1, "name": "b", "status": "SUCCEEDED", "result": "world"}
],
"completionReason": "ALL_COMPLETED"
})
.to_string()
}
#[tokio::test]
async fn abandoned_owner_replays_no_batch() {
use crate::future::Branch;
let bound = std::time::Duration::from_secs(1);
for parallel in [false, true] {
let log = CheckpointLog::from_records(vec![succeeded_record(
"1-1",
&terminal_batch_payload(),
)]);
let (root, client) = test_ctx_with_client(log);
let replay = |ctx: DurableContext| async move {
if parallel {
ctx.parallel(vec![
Branch::new("a", |_c| async { Ok(String::new()) }),
Branch::new("b", |_c| async { Ok(String::new()) }),
])
.await_batch()
.await
} else {
ctx.map(vec![0_i32, 1], |_c, _item, _idx| async {
Ok(String::new())
})
.await_batch()
.await
}
};
let control = root.new_child("1", None);
let replayed = tokio::time::timeout(bound, replay(control))
.await
.expect("the recorded batch replays")
.expect("the replay succeeds");
assert_eq!(
replayed.results().collect::<Vec<_>>(),
[&"hello".to_owned(), &"world".to_owned()]
);
let item = root.new_child("1", None);
root.admission()
.abandon(item.owners().iter().map(String::as_str));
let replayed = tokio::time::timeout(bound, replay(item)).await;
assert!(
replayed.is_err(),
"parallel {parallel}: the recorded batch is not replayed for an abandoned owner"
);
assert!(client.recorded_updates().is_empty(), "nothing is sent");
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn abandoned_item_terminal_write_is_rejected() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let tasks = Arc::clone(ctx.coordinator().tasks());
let entered = Arc::new(tokio::sync::Notify::new());
let (release, released) = std::sync::mpsc::channel::<()>();
let released = Arc::new(std::sync::Mutex::new(Some(released)));
let body = {
let entered = Arc::clone(&entered);
move |_child: DurableContext, item: i32, _idx: usize| {
let entered = Arc::clone(&entered);
let released = Arc::clone(&released);
async move {
if item == 0 {
entered.notified().await;
return Ok(0_i32);
}
let released = released.lock().unwrap().take().expect("item 1 runs once");
entered.notify_one();
tokio::task::block_in_place(|| {
released.recv().expect("the test releases item 1");
});
Ok(1_i32)
}
}
};
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.map(vec![0_i32, 1], body)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch does not wait for its running item")
.expect("the batch succeeds");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
assert_eq!(item_action_count(&client, 0, &OperationAction::Succeed), 1);
release.send(()).expect("item 1 waits");
let mut ended = false;
for _ in 0..500 {
if tasks.live() == 0 {
ended = true;
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert!(ended, "item 1's task ended");
assert_eq!(
item_action_count(&client, 1, &OperationAction::Succeed),
0,
"the abandoned item wrote no terminal update"
);
}
#[tokio::test(start_paused = true)]
async fn early_completion_abandons_parked_item() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let slot: Arc<std::sync::Mutex<Option<crate::DurableFuture<i32>>>> =
Arc::new(std::sync::Mutex::new(None));
let ran = no_guard();
let body = {
let slot = Arc::clone(&slot);
let ran = Arc::clone(&ran);
move |child: DurableContext, item: i32, _idx: usize| {
let slot = Arc::clone(&slot);
let ran = Arc::clone(&ran);
async move {
if item == 0 {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
return Ok(0_i32);
}
let escaped = child
.step(move |_| async move {
ran.store(true, Ordering::SeqCst);
Ok(5_i32)
})
.name("escaped")
.future();
*slot.lock().unwrap() = Some(escaped);
child
.wait(std::time::Duration::from_secs(5))
.name("park")
.await?;
Ok(1_i32)
}
}
};
let batch = tokio::time::timeout(
EARLY_BOUND,
ctx.hold_activity(
ctx.map(vec![0_i32, 1], body)
.completion(
crate::builders::map_parallel::CompletionConfig::with_min_successful(1),
)
.await_batch(),
),
)
.await
.expect("the batch completes")
.expect("the batch succeeds");
assert_eq!(
statuses(&batch),
[
(0, BatchItemStatus::Succeeded),
(1, BatchItemStatus::Started)
]
);
assert!(has_start(&client, "park"), "item 1 parked on its wait");
let [_, parked] = item_wires();
assert!(
!ctx.admission().admits(&[parked]),
"the parked item is abandoned"
);
let escaped = slot
.lock()
.unwrap()
.take()
.expect("item 1 stored its step future");
let awaited = tokio::time::timeout(std::time::Duration::from_secs(1), escaped).await;
assert!(awaited.is_err(), "the escaped step never resolves");
assert!(!ran.load(Ordering::SeqCst), "the escaped step never ran");
assert!(
!has_start(&client, "escaped"),
"the escaped step wrote nothing"
);
}
#[derive(Clone, Default)]
struct CallLog(Arc<std::sync::Mutex<Vec<(&'static str, String)>>>);
impl CallLog {
fn calls(&self) -> Vec<(&'static str, String)> {
let mut calls = self.0.lock().expect("lock").clone();
calls.sort();
calls
}
}
impl<T> Serdes<T> for CallLog
where
T: serde::Serialize + serde::de::DeserializeOwned + Send + 'static,
{
fn serialize(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
self.0
.lock()
.expect("lock")
.push(("serialize", context.operation_id().to_owned()));
std::future::ready(serde_json::to_string(&value).map_err(Into::into))
}
fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> impl Future<Output = Result<T, BoxError>> + Send {
self.0
.lock()
.expect("lock")
.push(("deserialize", context.operation_id().to_owned()));
std::future::ready(serde_json::from_str(&wire).map_err(Into::into))
}
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
struct Order {
id: u32,
sku: String,
}
struct OrderSerdes;
impl Serdes<Order> for OrderSerdes {
fn serialize(
&self,
value: Order,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(
serde_json::to_string(&value)
.map(|json| format!("order:{json}"))
.map_err(Into::into),
)
}
fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> impl Future<Output = Result<Order, BoxError>> + Send {
std::future::ready(
wire.strip_prefix("order:")
.ok_or_else(|| BoxError::from("missing order: prefix"))
.and_then(|json| serde_json::from_str(json).map_err(Into::into)),
)
}
}
fn order(id: u32) -> Order {
Order {
id,
sku: format!("sku-{id}"),
}
}
fn wire_of(positional: &str) -> String {
crate::engine::compute_wire_id_public(positional)
}
fn parent_payload_json(client: &crate::client::InMemoryExecutionClient) -> serde_json::Value {
serde_json::from_str(&parent_payload(client)).expect("the batch payload is JSON")
}
fn json_serdes_record_json(
client: &crate::client::InMemoryExecutionClient,
) -> serde_json::Value {
let payload = parent_payload(client);
let record_text: String =
serde_json::from_str(&payload).expect("JsonSerdes stores a quoted JSON string");
record_json(&record_text)
}
fn terminal_batch_log(payload: &str) -> CheckpointLog {
CheckpointLog::from_records(vec![succeeded_record("1", payload)])
}
fn js_record(record_text: &str) -> JsBatchRecord {
parse_batch_record(record_text).expect("the record parses")
}
#[test]
fn batch_checkpoint_item_round_trips_empty_result() {
let result: BatchResult<()> = BatchResult::new(
vec![BatchItem::new(
0,
"map-item-0".to_owned(),
BatchItemStatus::Succeeded,
Some(()),
None,
None,
)],
CompletionReason::AllCompleted,
);
let json = from_batch_result(&result, &[Some(String::new())], &ItemEncoding::AllSerdes);
assert!(json.contains(r#""result":"""#), "json: {json}");
let back = js_record(&json);
assert_eq!(
back.all[0].result.as_deref().map(RawValue::get),
Some(r#""""#)
);
}
#[test]
fn batch_checkpoint_started_item_has_no_result_or_error() {
let started: BatchResult<u32> = BatchResult::new(
vec![BatchItem::new(
0,
"map-item-0".to_owned(),
BatchItemStatus::Started,
None,
None,
None,
)],
CompletionReason::MinSuccessfulReached,
);
let json = record_json(&from_batch_result(
&started,
&[None],
&ItemEncoding::AllJson,
));
let item = json.pointer("/all/0").expect("one item");
assert!(item.get("result").is_none(), "item: {item}");
assert!(item.get("error").is_none(), "item: {item}");
assert_eq!(
json,
serde_json::json!({
"all": [{"index": 0, "status": "STARTED", "name": "map-item-0"}],
"completionReason": "MIN_SUCCESSFUL_REACHED",
})
);
}
#[tokio::test]
async fn map_struct_values_are_embedded_as_json() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let out = ctx
.map(vec![1_u32, 2], |_child, item, _idx| async move {
Ok(order(item))
})
.await
.expect("map succeeds");
assert_eq!(out, vec![order(1), order(2)]);
let payload = parent_payload_json(&client);
for (i, id) in [1_u32, 2].into_iter().enumerate() {
assert_eq!(
payload["all"][i]["result"],
serde_json::to_value(order(id)).expect("JSON"),
"payload: {payload}"
);
}
}
#[tokio::test]
async fn map_custom_serdes_values_are_strings() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let out = ctx
.map(vec![1_u32, 2], |_child, item, _idx| async move {
Ok(order(item))
})
.serdes(OrderSerdes)
.await
.expect("map succeeds");
assert_eq!(out, vec![order(1), order(2)]);
let mut child_payloads = child_success_payloads(&client);
child_payloads.sort();
let payload = parent_payload_json(&client);
let mut results: Vec<String> = payload["all"]
.as_array()
.expect("all is an array")
.iter()
.map(|item| item["result"].as_str().expect("a string result").to_owned())
.collect();
results.sort();
assert_eq!(results, child_payloads);
assert!(results.iter().all(|r| r.starts_with("order:")));
}
#[tokio::test]
async fn result_serdes_keeps_the_custom_item_encoding() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
ctx.map(
vec![1_u32],
|_child, item, _idx| async move { Ok(order(item)) },
)
.serdes(OrderSerdes)
.result_serdes(crate::serdes::JsonSerdes)
.await
.expect("map succeeds");
assert_eq!(
json_serdes_record_json(&client)["all"][0]["result"],
serde_json::json!(r#"order:{"id":1,"sku":"sku-1"}"#)
);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
ctx.parallel(vec![crate::future::Branch::new("a", |_c| async {
Ok(order(1))
})])
.serdes(OrderSerdes)
.result_serdes(crate::serdes::JsonSerdes)
.await
.expect("parallel succeeds");
assert_eq!(
json_serdes_record_json(&client)["all"][0]["result"],
serde_json::json!(r#"order:{"id":1,"sku":"sku-1"}"#)
);
}
fn typed_declined() -> BoxError {
Box::new(crate::TypedError::with_type("PaymentError", "declined").with_error_data("d"))
}
struct TypedItemSerializeFailure;
impl Serdes<u32> for TypedItemSerializeFailure {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
_value: u32,
_context: SerdesContext,
) -> Result<String, BoxError> {
Err(typed_declined())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<u32, BoxError> {
Ok(wire.parse()?)
}
}
struct TypedItemDeserializeFailure;
impl Serdes<u32> for TypedItemDeserializeFailure {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(&self, value: u32, _context: SerdesContext) -> Result<String, BoxError> {
Ok(value.to_string())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
_wire: String,
_context: SerdesContext,
) -> Result<u32, BoxError> {
Err(typed_declined())
}
}
struct TypedResultDeserializeFailure;
impl Serdes<String> for TypedResultDeserializeFailure {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(value)
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
_wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Err(typed_declined())
}
}
#[tokio::test]
async fn typed_item_serialize_failure_records_the_wrapped_message() {
const MESSAGE: &str =
"operation error: child_context: internal error: serialize result: declined";
let assert_item = |batch: &BatchResult<u32>| {
let item = &batch.items[0];
assert_eq!(item.status, BatchItemStatus::Failed);
assert_eq!(
item.error_type.as_deref(),
Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE)
);
assert_eq!(item.error_message.as_deref(), Some(MESSAGE));
assert_eq!(item.error_data, None);
assert!(item.stack_trace.is_empty(), "{item:?}");
};
let tolerate_one =
|| crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1);
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let live = ctx
.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.serdes(TypedItemSerializeFailure)
.completion(tolerate_one())
.await_batch()
.await
.expect("a tolerated failure is a batch result");
assert_item(&live);
let payload = parent_payload(&client);
let replayed = test_ctx(terminal_batch_log(&payload))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.serdes(TypedItemSerializeFailure)
.completion(tolerate_one())
.await_batch()
.await
.expect("replay succeeds");
assert_item(&replayed);
}
#[tokio::test]
async fn typed_result_deserialize_failure_records_the_wrapped_message() {
let expected = crate::error::WireError::new(
Some("ChildContextError"),
Some("operation error: child_context: internal error: deserialize result: declined"),
);
let assert_error = |err: &OperationError| {
let OperationErrorKind::ChildContext(child) = err.kind() else {
unreachable!("expected a child context error, got {:?}", err.kind());
};
assert!(
matches!(child.kind(), ChildContextErrorKind::Internal),
"{err:?}"
);
assert_eq!(
std::error::Error::source(child)
.map(ToString::to_string)
.as_deref(),
Some("deserialize result: declined")
);
assert_eq!(err.wire(), None);
assert_eq!(crate::error::wire_error_for(err, "Error"), expected);
};
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let live = ctx
.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.result_serdes(TypedResultDeserializeFailure)
.await
.expect("the live run returns its items");
assert_eq!(live, [1]);
let payload = parent_payload(&client);
let replayed: Result<Vec<u32>, OperationError> = test_ctx(terminal_batch_log(&payload))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.result_serdes(TypedResultDeserializeFailure)
.await;
let replayed = replayed.expect_err("the recorded batch cannot be decoded");
assert_error(&replayed);
}
#[tokio::test]
async fn typed_item_deserialize_failure_records_the_wrapped_message() {
let expected = crate::error::WireError::new(
Some("ChildContextError"),
Some("operation error: child_context: internal error: deserialize result: declined"),
);
let assert_error = |err: &OperationError| {
let OperationErrorKind::ChildContext(child) = err.kind() else {
unreachable!("expected a child context error, got {:?}", err.kind());
};
assert!(
matches!(child.kind(), ChildContextErrorKind::Internal),
"{err:?}"
);
assert_eq!(
std::error::Error::source(child)
.map(ToString::to_string)
.as_deref(),
Some("deserialize result: declined")
);
assert_eq!(err.wire(), None);
assert_eq!(crate::error::wire_error_for(err, "Error"), expected);
};
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let live: Result<Vec<u32>, OperationError> = ctx
.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.serdes(TypedItemDeserializeFailure)
.await;
let live = live.expect_err("the item value cannot be decoded");
assert_error(&live);
assert_eq!(child_success_payloads(&client), ["1"]);
let replayed: Result<Vec<u32>, OperationError> = test_ctx(CheckpointLog::from_records(
vec![succeeded_record("1-1", "1")],
))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a recorded item replays")
})
.serdes(TypedItemDeserializeFailure)
.await;
let replayed = replayed.expect_err("the recorded item cannot be decoded");
assert_error(&replayed);
}
#[tokio::test]
async fn failed_item_error_has_js_shape() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let live = ctx
.map(vec![1_u32], |_child, _item, _idx| async move {
Err::<u32, BoxError>(Box::new(
crate::TypedError::with_type("PaymentError", "declined").with_error_data("d"),
))
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated failure is a batch result");
assert_eq!(live.items[0].error_message.as_deref(), Some("declined"));
let payload = parent_payload(&client);
let json: serde_json::Value = serde_json::from_str(&payload).expect("JSON");
assert_eq!(
json["all"][0]["error"],
serde_json::json!({
"ErrorType": "ChildContextError",
"ErrorMessage": "declined",
"ErrorData": "d",
"Cause": {"ErrorType": "PaymentError", "ErrorMessage": "declined", "ErrorData": "d"},
})
);
let replayed: BatchResult<u32> = test_ctx(terminal_batch_log(&payload))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("replay succeeds");
assert_eq!(
replayed.items[0].error_type.as_deref(),
Some("PaymentError")
);
assert_eq!(replayed.items[0].error_data.as_deref(), Some("d"));
assert_eq!(replayed.items[0].error_message.as_deref(), Some("declined"));
}
#[tokio::test]
async fn failed_item_without_recorded_type_falls_back_to_error() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
ctx.map(vec![1_u32], |_child, _item, _idx| async move {
Err::<u32, BoxError>("boom".into())
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated failure is a batch result");
let payload = parent_payload(&client);
let json: serde_json::Value = serde_json::from_str(&payload).expect("JSON");
assert_eq!(json["all"][0]["error"]["Cause"]["ErrorType"], "Error");
let replayed: BatchResult<u32> = test_ctx(terminal_batch_log(&payload))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("replay succeeds");
assert_eq!(replayed.items[0].error_type.as_deref(), Some("Error"));
}
#[tokio::test]
async fn js_error_without_cause_reads_top_level() {
let payload = serde_json::json!({
"all": [{
"index": 0,
"status": "FAILED",
"name": "map-item-0",
"error": {
"ErrorType": "TopError",
"ErrorMessage": "top",
"ErrorData": "top-data",
"StackTrace": ["top-frame"],
},
"result": 5,
}],
"completionReason": "ALL_COMPLETED",
});
let replayed: BatchResult<u32> = test_ctx(terminal_batch_log(&payload.to_string()))
.map(vec![1_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("replay succeeds");
let item = &replayed.items[0];
assert_eq!(item.error_type.as_deref(), Some("TopError"));
assert_eq!(item.error_message.as_deref(), Some("top"));
assert_eq!(item.error_data.as_deref(), Some("top-data"));
assert_eq!(item.stack_trace, ["top-frame".to_owned()]);
assert_eq!(item.result, None);
}
#[tokio::test]
async fn results_reason_record_fails_replay() {
let payload = r#"{"results":[{"index":0,"name":"a","status":"SUCCEEDED","result":"\"hello\""}],"reason":"ALL_COMPLETED"}"#;
let err = test_ctx(terminal_batch_log(payload))
.map(vec![0_u32], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.item_namer(|_, _| "a".to_owned())
.await_batch()
.await
.map(|batch: BatchResult<String>| batch.items.len())
.expect_err("the record fails replay");
assert!(
format!("{err:#}").contains("batch record has neither `all` nor `completionReason`"),
"unexpected error: {err:#}"
);
}
fn record_of_wires(wires: &[&str], encoding: &ItemEncoding) -> String {
let items = (0..wires.len())
.map(|index| {
BatchItem::new(
index,
format!("map-item-{index}"),
BatchItemStatus::Succeeded,
Some(()),
None,
None,
)
})
.collect();
let result: BatchResult<()> = BatchResult::new(items, CompletionReason::AllCompleted);
let wires: Vec<Option<String>> = wires.iter().map(|w| Some((*w).to_owned())).collect();
from_batch_result(&result, &wires, encoding)
}
fn result_texts(record_text: &str) -> Vec<Option<String>> {
js_record(record_text)
.all
.iter()
.map(|item| item.result.as_deref().map(|raw| raw.get().to_owned()))
.collect()
}
#[derive(Clone, Default)]
struct WireLog(Arc<std::sync::Mutex<Vec<(String, String)>>>);
impl WireLog {
fn calls(&self) -> Vec<(String, String)> {
self.0.lock().expect("lock").clone()
}
}
impl Serdes<String> for WireLog {
fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Ok(value))
}
fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
self.0
.lock()
.expect("lock")
.push((context.operation_id().to_owned(), wire.clone()));
std::future::ready(Ok(wire))
}
}
#[tokio::test]
async fn record_round_trip_keeps_exact_json_item_text() {
let wires = [
"null",
r#""a string""#,
"[1,2,3]",
r#"{"n":340282366920938463463374607431768211455,"m":{"k":-170141183460469231731687303715884105728}}"#,
"1e2",
"1.0",
"-0",
];
let text = record_of_wires(&wires, &ItemEncoding::AllJson);
for wire in wires {
assert!(
text.contains(&format!(r#""result":{wire}}}"#)),
"the record embeds {wire} unchanged: {text}"
);
}
assert_eq!(
result_texts(&text),
wires
.iter()
.map(|w| Some((*w).to_owned()))
.collect::<Vec<_>>()
);
let log = WireLog::default();
let ctx = test_ctx(CheckpointLog::empty());
let replayed: BatchResult<String> = to_batch_result(
&ctx,
&text,
&Arc::new(log.clone()),
ItemEncoding::AllJson,
wires.len(),
"arn:test",
&ItemIds::new(&OperationId::new("1".to_owned()), wires.len()),
)
.await
.expect("the record replays");
let values: Vec<String> = replayed
.items
.into_iter()
.map(|item| item.result.expect("a value"))
.collect();
assert_eq!(values, wires);
let handed: Vec<String> = log.calls().into_iter().map(|(_, wire)| wire).collect();
assert_eq!(handed, wires);
}
#[tokio::test]
async fn custom_serdes_item_records_a_json_string() {
let wires = [
"",
r#"{"a":1}"#,
"340282366920938463463374607431768211455",
"plain",
];
let text = record_of_wires(&wires, &ItemEncoding::AllSerdes);
assert_eq!(
result_texts(&text),
vec![
Some(r#""""#.to_owned()),
Some(r#""{\"a\":1}""#.to_owned()),
Some(r#""340282366920938463463374607431768211455""#.to_owned()),
Some(r#""plain""#.to_owned()),
]
);
let replayed: BatchResult<String> = to_batch_result(
&test_ctx(CheckpointLog::empty()),
&text,
&Arc::new(WireLog::default()),
ItemEncoding::AllSerdes,
wires.len(),
"arn:test",
&ItemIds::new(&OperationId::new("1".to_owned()), wires.len()),
)
.await
.expect("the record replays");
let values: Vec<String> = replayed
.items
.into_iter()
.map(|item| item.result.expect("a value"))
.collect();
assert_eq!(values, wires);
}
fn record_error(text: &str) -> String {
parse_batch_record(text).expect_err("the record must be rejected")
}
#[test]
fn record_parser_reads_the_js_form() {
let js = r#"{"all":[{"index":0,"status":"SUCCEEDED","name":"a","result":7}],"completionReason":"ALL_COMPLETED"}"#;
let record = parse_batch_record(js).expect("the JS form parses");
assert_eq!(record.completion_reason, "ALL_COMPLETED");
assert_eq!(
record
.all
.first()
.and_then(|i| i.result.as_deref())
.map(RawValue::get),
Some("7")
);
}
#[test]
fn record_parser_rejects_non_records() {
assert!(record_error(r#"{"all":"#).contains("batch record is not valid"));
let trailing = record_error(r#"{"all":[],"completionReason":"ALL_COMPLETED"} {}"#);
assert!(trailing.contains("trailing characters"), "{trailing}");
for text in [
"[]",
r#"[{"all":[],"completionReason":"ALL_COMPLETED"}]"#,
"5",
r#""s""#,
"null",
"true",
] {
let err = record_error(text);
assert!(
err.contains("expected a batch record object"),
"{text}: {err}"
);
}
}
#[test]
fn record_parser_requires_both_fields() {
let neither = "batch record has neither `all` nor `completionReason`";
let cases = [
("{}", neither),
(r#"{"results":[],"reason":"ALL_COMPLETED"}"#, neither),
(
r#"{"all":[]}"#,
"batch record has `all` without `completionReason`",
),
(
r#"{"completionReason":"ALL_COMPLETED"}"#,
"batch record has `completionReason` without `all`",
),
];
for (text, expected) in cases {
assert_eq!(record_error(text), expected, "{text}");
}
}
#[test]
fn record_parser_reads_null_fields_as_absent() {
let cases = [
(
r#"{"all":null,"completionReason":"ALL_COMPLETED"}"#,
"batch record has `completionReason` without `all`",
),
(
r#"{"all":[],"completionReason":null}"#,
"batch record has `all` without `completionReason`",
),
];
for (text, expected) in cases {
assert_eq!(record_error(text), expected, "{text}");
}
}
#[test]
fn record_parser_rejects_wrong_field_types() {
for (text, field_type) in [
(
r#"{"all":5,"completionReason":"ALL_COMPLETED"}"#,
"invalid type: integer `5`, expected a sequence",
),
(
r#"{"all":[],"completionReason":5}"#,
"invalid type: integer `5`, expected a string",
),
] {
let err = record_error(text);
assert!(err.contains(field_type), "{text}: {err}");
}
}
#[test]
fn record_parser_rejects_duplicate_fields() {
for (text, field) in [
(
r#"{"all":[],"all":[],"completionReason":"ALL_COMPLETED"}"#,
"all",
),
(
r#"{"all":null,"all":[],"completionReason":"ALL_COMPLETED"}"#,
"all",
),
(
r#"{"all":[],"completionReason":"ALL_COMPLETED","completionReason":"ALL_COMPLETED"}"#,
"completionReason",
),
(
r#"{"all":[],"completionReason":null,"completionReason":"ALL_COMPLETED"}"#,
"completionReason",
),
] {
let err = record_error(text);
assert!(
err.contains(&format!("duplicate field `{field}`")),
"{text}: {err}"
);
}
}
#[test]
fn record_parser_ignores_unknown_fields() {
for text in [
r#"{"extra":{"nested":[1,{"a":null}]},"all":[],"completionReason":"ALL_COMPLETED","type":"MapResult"}"#,
r#"{"all":[],"completionReason":"ALL_COMPLETED","results":"x","reason":5}"#,
] {
assert!(parse_batch_record(text).is_ok(), "{text}");
}
}
async fn js_replay_error(payload: &serde_json::Value, custom: bool) -> String {
async fn never_runs(
_child: DurableContext,
_item: u32,
_idx: usize,
) -> Result<u32, BoxError> {
unreachable!("a terminal batch replays")
}
let ctx = test_ctx(terminal_batch_log(&payload.to_string()));
let map = ctx.map(vec![1_u32], never_runs);
let err = if custom {
map.serdes(CallLog::default()).await_batch().await
} else {
map.await_batch().await
}
.expect_err("the record fails replay");
crate::error::chain_string(&err)
}
#[tokio::test]
async fn malformed_stored_record_fails_replay() {
let payload = serde_json::json!({"completionReason": "ALL_COMPLETED"});
let err = js_replay_error(&payload, false).await;
assert!(
err.contains("batch record has `completionReason` without `all`"),
"{err}"
);
}
#[tokio::test]
async fn serdes_item_with_non_string_result_fails_replay() {
let payload = serde_json::json!({
"all": [{"index": 0, "status": "SUCCEEDED", "name": "map-item-0", "result": 1}],
"completionReason": "ALL_COMPLETED",
});
let err = js_replay_error(&payload, true).await;
assert!(
err.contains("batch record item 0: expected a string result"),
"{err}"
);
}
#[tokio::test]
async fn js_item_without_name_fails_replay() {
let payload = serde_json::json!({
"all": [{"index": 0, "status": "SUCCEEDED", "result": 1}],
"completionReason": "ALL_COMPLETED",
});
let err = js_replay_error(&payload, false).await;
assert!(err.contains("batch record item 0: missing name"), "{err}");
}
#[tokio::test]
async fn js_item_with_unknown_status_fails_replay() {
let payload = serde_json::json!({
"all": [{"index": 0, "status": "WEIRD", "name": "map-item-0"}],
"completionReason": "ALL_COMPLETED",
});
let err = js_replay_error(&payload, false).await;
assert!(err.contains("unknown item status: WEIRD"), "{err}");
}
#[tokio::test]
async fn oversized_map_writes_js_summary() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let big = "x".repeat(101 * 1024);
ctx.map(vec![0_u32, 1, 2], move |_child, _item, _idx| {
let big = big.clone();
async move { Ok(big) }
})
.await
.expect("map succeeds");
let parent = parent_succeed_update(&client);
assert_eq!(
parent.context_options().and_then(|c| c.replay_children),
Some(true)
);
assert_eq!(
parent_payload_json(&client),
serde_json::json!({
"type": "MapResult",
"totalCount": 3,
"successCount": 3,
"failureCount": 0,
"completionReason": "ALL_COMPLETED",
"status": "SUCCEEDED",
"itemStatuses": "SSS",
})
);
}
#[test]
fn oversized_summary_text_has_js_key_order() {
let result = BatchResult::new(
vec![BatchItem::<u32>::new(
0,
"a".to_owned(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
)],
CompletionReason::AllCompleted,
);
assert_eq!(
oversized_batch_summary(&result, MAP_SUB_TYPE, None),
r#"{"type":"MapResult","totalCount":1,"successCount":1,"failureCount":0,"completionReason":"ALL_COMPLETED","status":"SUCCEEDED","itemStatuses":"S"}"#
);
assert_eq!(
oversized_batch_summary(&result, PARALLEL_SUB_TYPE, None),
r#"{"type":"ParallelResult","totalCount":1,"successCount":1,"failureCount":0,"startedCount":0,"completionReason":"ALL_COMPLETED","status":"SUCCEEDED","itemStatuses":"S"}"#
);
assert_eq!(
oversized_batch_summary(&result, MAP_SUB_TYPE, Some("ok")),
r#"{"type":"MapResult","totalCount":1,"successCount":1,"failureCount":0,"completionReason":"ALL_COMPLETED","status":"SUCCEEDED","itemStatuses":"S","summary":"ok"}"#
);
}
#[test]
fn caller_summary_is_last_key() {
let result = BatchResult::new(
vec![BatchItem::<u32>::new(
0,
"a".to_owned(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
)],
CompletionReason::AllCompleted,
);
let plain = oversized_batch_summary(&result, MAP_SUB_TYPE, None);
let text = oversized_batch_summary(&result, MAP_SUB_TYPE, Some("ok \"1\""));
assert!(text.ends_with(r#","summary":"ok \"1\""}"#), "{text}");
let mut value: serde_json::Value = serde_json::from_str(&text).expect("JSON");
assert_eq!(value["summary"], "ok \"1\"");
value.as_object_mut().expect("an object").remove("summary");
assert_eq!(
value,
serde_json::from_str::<serde_json::Value>(&plain).expect("JSON")
);
}
#[tokio::test]
async fn oversized_map_with_failure_writes_failed_status() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let big = "x".repeat(300 * 1024);
ctx.map(vec![0_u32, 1], move |_child, item, _idx| {
let big = big.clone();
async move {
if item == 0 {
Ok(big)
} else {
Err("boom".into())
}
}
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(1),
)
.await_batch()
.await
.expect("a tolerated failure is a batch result");
let summary = parent_payload_json(&client);
assert_eq!(summary["status"], "FAILED", "summary: {summary}");
assert_eq!(summary["itemStatuses"], "SF");
assert_eq!(summary["failureCount"], 1);
}
#[tokio::test]
async fn oversized_parallel_writes_started_count() {
use crate::future::Branch;
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let big = "x".repeat(300 * 1024);
let branches = vec![
Branch::new("big", move |_c| async move { Ok(big) }),
Branch::new("slow", |c| async move {
Ok(
c.step(|_| std::future::pending::<Result<String, BoxError>>())
.name("forever")
.await?,
)
}),
];
let batch = tokio::time::timeout(
std::time::Duration::from_secs(10),
ctx.parallel(branches)
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(1))
.await_batch(),
)
.await
.expect("the batch does not wait for the running branch")
.expect("the batch succeeds");
assert_eq!(batch.items[1].status, BatchItemStatus::Started);
assert_eq!(
parent_payload_json(&client),
serde_json::json!({
"type": "ParallelResult",
"totalCount": 2,
"successCount": 1,
"failureCount": 0,
"startedCount": 1,
"completionReason": "MIN_SUCCESSFUL_REACHED",
"status": "SUCCEEDED",
"itemStatuses": "S-",
})
);
}
#[test]
fn oversized_summary_status_follows_custom_completion() {
let failed_item = || {
BatchItem::<u32>::new(
0,
"a".to_owned(),
BatchItemStatus::Failed,
None,
Some("boom".to_owned()),
None,
)
};
let succeeded_item = || {
BatchItem::<u32>::new(
0,
"a".to_owned(),
BatchItemStatus::Succeeded,
Some(1),
None,
None,
)
};
let summary = |items, reason| -> serde_json::Value {
let result = BatchResult::new(items, reason);
serde_json::from_str(&oversized_batch_summary(&result, MAP_SUB_TYPE, None))
.expect("the summary is JSON")
};
let custom_ok = summary(
vec![failed_item()],
CompletionReason::CustomCompletionSucceeded,
);
assert_eq!(custom_ok["status"], "SUCCEEDED", "summary: {custom_ok}");
assert_eq!(custom_ok["itemStatuses"], "F");
assert_eq!(custom_ok["completionReason"], "CUSTOM_COMPLETION_SUCCEEDED");
let custom_failed = summary(
vec![succeeded_item()],
CompletionReason::CustomCompletionFailed,
);
assert_eq!(
custom_failed["status"], "FAILED",
"summary: {custom_failed}"
);
assert_eq!(custom_failed["itemStatuses"], "S");
assert_eq!(
custom_failed["completionReason"],
"CUSTOM_COMPLETION_FAILED"
);
}
#[test]
fn failed_item_writes_stack_trace_at_both_levels() {
let mut item = BatchItem::<u32>::new(
0,
"a".to_owned(),
BatchItemStatus::Failed,
None,
Some("it broke".to_owned()),
Some("CustomFailure".to_owned()),
);
item.error_data = Some("data".to_owned());
item.stack_trace = vec!["frame-a".to_owned(), "frame-b".to_owned()];
let result = BatchResult::new(vec![item], CompletionReason::AllCompleted);
let payload = record_json(&from_batch_result(&result, &[None], &ItemEncoding::AllJson));
assert_eq!(
payload.pointer("/all/0/error"),
Some(&serde_json::json!({
"ErrorType": "ChildContextError",
"ErrorMessage": "it broke",
"ErrorData": "data",
"StackTrace": ["frame-a", "frame-b"],
"Cause": {
"ErrorType": "CustomFailure",
"ErrorMessage": "it broke",
"ErrorData": "data",
"StackTrace": ["frame-a", "frame-b"],
},
}))
);
}
#[tokio::test]
async fn js_error_prefers_cause_and_falls_back_per_field() {
let payload = serde_json::json!({
"all": [
{
"index": 0,
"status": "FAILED",
"name": "map-item-0",
"error": {
"ErrorType": "TopError",
"ErrorMessage": "top",
"ErrorData": "top-data",
"StackTrace": ["top-frame"],
"Cause": {
"ErrorType": "CauseError",
"ErrorMessage": "cause",
"ErrorData": "cause-data",
"StackTrace": ["cause-frame"],
},
},
},
{
"index": 1,
"status": "FAILED",
"name": "map-item-1",
"error": {
"ErrorType": "TopError",
"ErrorMessage": "top",
"ErrorData": "top-data",
"StackTrace": ["top-frame"],
"Cause": {},
},
},
],
"completionReason": "ALL_COMPLETED",
});
let replayed: BatchResult<u32> = test_ctx(terminal_batch_log(&payload.to_string()))
.map(vec![1_u32, 2], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(2),
)
.await_batch()
.await
.expect("replay succeeds");
let fields = |item: &BatchItem<u32>| {
(
item.error_type.clone(),
item.error_message.clone(),
item.error_data.clone(),
item.stack_trace.clone(),
)
};
assert_eq!(
fields(&replayed.items[0]),
(
Some("CauseError".to_owned()),
Some("cause".to_owned()),
Some("cause-data".to_owned()),
vec!["cause-frame".to_owned()],
)
);
assert_eq!(
fields(&replayed.items[1]),
(
Some("TopError".to_owned()),
Some("top".to_owned()),
Some("top-data".to_owned()),
Vec::new(),
)
);
}
async fn js_replay_error_over(
payload: &serde_json::Value,
inputs: u32,
) -> (String, Vec<(&'static str, String)>) {
async fn never_runs(
_child: DurableContext,
_item: u32,
_idx: usize,
) -> Result<u32, BoxError> {
unreachable!("a terminal batch replays")
}
let calls = CallLog::default();
let err = test_ctx(terminal_batch_log(&payload.to_string()))
.map((0..inputs).collect::<Vec<_>>(), never_runs)
.serdes(calls.clone())
.await_batch()
.await
.expect_err("the record fails replay");
(crate::error::chain_string(&err), calls.calls())
}
fn js_record_with_indexes(indexes: &[usize]) -> serde_json::Value {
let all: Vec<_> = indexes
.iter()
.map(|index| {
serde_json::json!({
"index": index,
"status": "SUCCEEDED",
"name": format!("map-item-{index}"),
"result": "1",
})
})
.collect();
serde_json::json!({"all": all, "completionReason": "ALL_COMPLETED"})
}
#[tokio::test]
async fn js_record_with_malformed_indexes_fails_replay() {
let cases: [(&[usize], &str); 4] = [
(&[0, 2], "batch record item at position 1 has index 2"),
(&[0, 0], "batch record item at position 1 has index 0"),
(&[1, 0], "batch record item at position 0 has index 1"),
(
&[usize::MAX],
"batch record item at position 0 has index 18446744073709551615",
),
];
for (indexes, expected) in cases {
let (err, calls) = js_replay_error_over(&js_record_with_indexes(indexes), 3).await;
assert!(err.contains(expected), "indexes {indexes:?}: {err}");
assert_eq!(calls, vec![], "indexes {indexes:?}");
}
}
#[tokio::test]
async fn js_record_longer_than_the_input_fails_replay() {
let (err, calls) = js_replay_error_over(&js_record_with_indexes(&[0, 1, 2, 3]), 3).await;
assert!(
err.contains("batch record has 4 items but the input has 3"),
"{err}"
);
assert_eq!(calls, vec![]);
let replayed: BatchResult<u32> = test_ctx(terminal_batch_log(
&js_record_with_indexes(&[0, 1, 2]).to_string(),
))
.map(vec![0_u32, 1, 2], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.serdes(CallLog::default())
.await_batch()
.await
.expect("a record as long as the input replays");
assert_eq!(replayed.items.len(), 3);
}
fn started_middle_log(payload: &str) -> CheckpointLog {
CheckpointLog::from_records(vec![
decision_record_parent("1", payload),
succeeded_record("1-1", "\"a\""),
succeeded_record("1-3", "\"c\""),
])
}
async fn replay_started_middle(log: CheckpointLog) -> BatchResult<String> {
test_ctx(log)
.map(vec![0_usize, 1, 2, 3], |_child, _item, _idx| async move {
unreachable!("replay must not re-run any item body")
})
.completion(crate::builders::map_parallel::CompletionConfig::with_min_successful(2))
.await_batch()
.await
.expect("replay succeeds")
}
fn started_middle_rows(
batch: &BatchResult<String>,
) -> Vec<(usize, BatchItemStatus, Option<String>)> {
batch
.items
.iter()
.map(|i| (i.index, i.status, i.result.clone()))
.collect()
}
const STARTED_MIDDLE: [(usize, BatchItemStatus, Option<&str>); 3] = [
(0, BatchItemStatus::Succeeded, Some("a")),
(1, BatchItemStatus::Started, None),
(2, BatchItemStatus::Succeeded, Some("c")),
];
fn started_middle_expected() -> Vec<(usize, BatchItemStatus, Option<String>)> {
STARTED_MIDDLE
.iter()
.map(|(i, s, r)| (*i, *s, r.map(str::to_owned)))
.collect()
}
#[tokio::test]
async fn python_decision_record_replays() {
let batch = replay_started_middle(started_middle_log(
r#"{"totalCount":3,"completionReason":"MIN_SUCCESSFUL_REACHED","startedIndexes":[1]}"#,
))
.await;
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(started_middle_rows(&batch), started_middle_expected());
}
fn depth_flag(update: &OperationUpdate) -> Option<bool> {
update
.context_options()
.and_then(aws_sdk_lambda::types::ContextOptions::replay_children)
}
fn depth_update(
client: &crate::client::InMemoryExecutionClient,
name: &str,
action: &OperationAction,
) -> OperationUpdate {
client
.recorded_updates()
.into_iter()
.find(|u| u.name() == Some(name) && u.action() == action)
.unwrap_or_else(|| unreachable!("no {action:?} update for {name}"))
}
async fn depth_map(depth: u32) -> Arc<crate::client::InMemoryExecutionClient> {
let (ctx, client) = DurableContext::test_context_with_depth(depth);
let out: Vec<u32> = ctx
.map(
vec![1_u32, 2],
|_child, item, _idx| async move { Ok(item * 10) },
)
.name("m")
.await
.expect("map succeeds");
assert_eq!(out, vec![10, 20]);
client
}
#[tokio::test]
async fn preserved_container_records_result_payload() {
let plain_client = depth_map(0).await;
let plain = depth_update(&plain_client, "m", &OperationAction::Succeed);
let client = depth_map(1).await;
let container = depth_update(&client, "m", &OperationAction::Succeed);
assert_eq!(depth_flag(&container), Some(true));
assert_eq!(depth_flag(&plain), None);
assert_eq!(container.payload(), plain.payload());
for item in ["map-item-0", "map-item-1"] {
let update = depth_update(&client, item, &OperationAction::Succeed);
assert_eq!(depth_flag(&update), None, "{item}");
}
}
#[tokio::test]
async fn depth_two_flags_items() {
let client = depth_map(2).await;
for (item, payload) in [("map-item-0", "10"), ("map-item-1", "20")] {
let update = depth_update(&client, item, &OperationAction::Succeed);
assert_eq!(depth_flag(&update), Some(true), "{item}");
assert_eq!(update.payload(), Some(payload), "{item}");
}
}
#[tokio::test]
async fn preserved_item_failures_have_flag() {
for (depth, flag) in [(2, Some(true)), (1, None)] {
let (ctx, client) = DurableContext::test_context_with_depth(depth);
let batch = ctx
.map(vec![0_u32], |_child, _item, _idx| async move {
Err::<u32, BoxError>("boom".into())
})
.name("m")
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(
1,
),
)
.await_batch()
.await
.expect("the batch tolerates the failure");
assert_eq!(batch.items[0].status, BatchItemStatus::Failed);
let fail = depth_update(&client, "map-item-0", &OperationAction::Fail);
assert_eq!(depth_flag(&fail), flag, "body failure at depth {depth}");
let (ctx, client) = DurableContext::test_context_with_depth(depth);
let batch = ctx
.map(vec![0_u32], |_child, _item, _idx| async move {
Ok("x".to_owned())
})
.name("m")
.serdes(FailingResultSerdes)
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_count(
1,
),
)
.await_batch()
.await
.expect("the batch tolerates the failure");
assert_eq!(batch.items[0].status, BatchItemStatus::Failed);
let fail = depth_update(&client, "map-item-0", &OperationAction::Fail);
assert_eq!(
depth_flag(&fail),
flag,
"serialization failure at depth {depth}"
);
}
}
#[derive(Debug)]
struct RejectsFirstMatch {
inner: Arc<crate::client::InMemoryExecutionClient>,
rejects: fn(&OperationUpdate) -> bool,
done: AtomicBool,
}
impl crate::client::ExecutionClient for RejectsFirstMatch {
fn checkpoint(
&self,
arn: &str,
token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::CheckpointOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
if updates.iter().any(|u| (self.rejects)(u)) && !self.done.swap(true, Ordering::SeqCst)
{
return Box::pin(async {
Err(crate::client::ExecutionClientError::new_non_retryable(
"rejected",
))
});
}
self.inner.checkpoint(arn, token, updates)
}
fn get_state(
&self,
arn: &str,
token: &str,
marker: Option<&str>,
) -> Pin<
Box<
dyn Future<
Output = Result<
crate::client::GetStateOutput,
crate::client::ExecutionClientError,
>,
> + Send
+ '_,
>,
> {
self.inner.get_state(arn, token, marker)
}
}
#[tokio::test]
async fn preserved_item_rejected_outcome_sends_no_replacement_fail() {
fn item_outcome(u: &OperationUpdate) -> bool {
u.name() == Some("map-item-0")
&& matches!(u.action(), OperationAction::Succeed | OperationAction::Fail)
}
for case in ["succeed", "fail", "serialize"] {
for depth in [2, 1] {
let inner = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
let client = Arc::new(RejectsFirstMatch {
inner: Arc::clone(&inner),
rejects: item_outcome,
done: AtomicBool::new(false),
});
let ctx = DurableContext::test_context_with_depth_and_client(depth, client);
let outcome = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::driver::test_support::drive_handler(ctx, None, move |ctx| async move {
let builder = ctx.map(vec![0_u32], move |_child, _item, _idx| async move {
if case == "fail" {
Err::<String, BoxError>("boom".into())
} else {
Ok("x".to_owned())
}
});
let result = if case == "serialize" {
builder.name("m").serdes(FailingResultSerdes).await
} else {
builder.name("m").await
};
result.map_err(crate::driver::test_support::wire_err)?;
Ok("done".to_owned())
}),
)
.await
.expect("the invocation ends within 5 s")
.expect("no panic");
assert!(
matches!(outcome, crate::driver::InvocationOutcome::Failed { .. }),
"{case}: {outcome:?}"
);
let accepted: Vec<OperationAction> = inner
.recorded_updates()
.into_iter()
.filter(|u| u.name() == Some("map-item-0"))
.map(|u| u.action().clone())
.collect();
assert_eq!(
accepted,
[OperationAction::Start],
"{case} at depth {depth}: no replacement FAIL"
);
}
}
}
#[tokio::test]
async fn preserved_full_result_replays_like_the_decision_record() {
let batch = replay_started_middle(started_middle_log(
r#"{"all":[{"index":0,"status":"SUCCEEDED","name":"map-item-0","result":"a"},{"index":1,"status":"STARTED","name":"map-item-1"},{"index":2,"status":"SUCCEEDED","name":"map-item-2","result":"c"}],"completionReason":"MIN_SUCCESSFUL_REACHED"}"#,
))
.await;
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(started_middle_rows(&batch), started_middle_expected());
}
#[tokio::test]
async fn preserved_full_result_rejects_unknown_item_status() {
let log = started_middle_log(
r#"{"all":[{"index":0,"status":"DONE","name":"map-item-0"}],"completionReason":"ALL_COMPLETED"}"#,
);
let result: Result<Vec<String>, _> = test_ctx(log)
.map(vec![0_usize, 1, 2, 3], |_child, _item, _idx| async move {
unreachable!("a corrupt record must not re-run any item")
})
.await;
let err = result.expect_err("an unknown status is corrupt");
assert!(
crate::error::chain_string(&err).contains("DONE"),
"{}",
crate::error::chain_string(&err)
);
}
#[tokio::test]
async fn preserved_full_result_rejects_misplaced_indexes() {
for all in [
r#"[{"index":0,"status":"SUCCEEDED","name":"map-item-0","result":"a"},{"index":0,"status":"STARTED","name":"map-item-0"}]"#,
r#"[{"index":1,"status":"STARTED","name":"map-item-1"},{"index":0,"status":"SUCCEEDED","name":"map-item-0","result":"a"}]"#,
] {
let log = started_middle_log(&format!(
r#"{{"all":{all},"completionReason":"MIN_SUCCESSFUL_REACHED"}}"#
));
let result: Result<Vec<String>, _> = test_ctx(log)
.map(vec![0_usize, 1, 2, 3], |_child, _item, _idx| async move {
unreachable!("a corrupt record must not re-run any item")
})
.await;
let err = result.expect_err("a misplaced index is corrupt");
assert!(
crate::error::chain_string(&err).contains("at position"),
"{all}: {}",
crate::error::chain_string(&err)
);
}
}
#[tokio::test]
async fn preserved_full_result_decode_failure_fails_replay() {
let log = started_middle_log(r#"{"all":[],"completionReason":"ALL_COMPLETED"}"#);
let result: Result<Vec<String>, _> = test_ctx(log)
.map(vec![0_usize, 1, 2, 3], |_child, _item, _idx| async move {
unreachable!("a payload that fails to decode must not re-run any item")
})
.result_serdes(FailingResultSerdes)
.await;
let err = result.expect_err("the decode fails");
assert!(
crate::error::chain_string(&err).contains("cannot decode"),
"{}",
crate::error::chain_string(&err)
);
}
#[tokio::test]
async fn js_summary_replays_like_the_decision_record() {
let batch = replay_started_middle(started_middle_log(
r#"{"type":"MapResult","totalCount":3,"successCount":2,"failureCount":0,"completionReason":"MIN_SUCCESSFUL_REACHED","status":"SUCCEEDED","itemStatuses":"S-S"}"#,
))
.await;
assert_eq!(batch.reason, CompletionReason::MinSuccessfulReached);
assert_eq!(started_middle_rows(&batch), started_middle_expected());
}
#[test]
fn replay_children_payload_is_read_in_order() {
let record = parse_replay_children_payload(Some(
r#"{"itemStatuses":"S-F","completionReason":"ALL_COMPLETED","totalCount":99}"#,
))
.expect("a JS summary parses")
.expect("a JS summary is a decision");
assert_eq!(record.total_count, 3, "the length of itemStatuses wins");
assert_eq!(record.completed_indexes, Some(vec![0, 2]));
assert_eq!(record.started_indexes, None);
assert_eq!(record.completion_reason, "ALL_COMPLETED");
let record = parse_replay_children_payload(Some(
r#"{"itemStatuses":"","completionReason":"ALL_COMPLETED"}"#,
))
.expect("an empty JS summary parses")
.expect("an empty JS summary is a decision");
assert_eq!(record.total_count, 0);
assert_eq!(record.completed_indexes, Some(vec![]));
for statuses in [r#""SX""#, "1", "null"] {
let payload =
format!(r#"{{"itemStatuses":{statuses},"completionReason":"ALL_COMPLETED"}}"#);
let err = parse_replay_children_payload(Some(&payload))
.expect_err("an invalid itemStatuses is corrupt");
assert!(
crate::error::chain_string(&err)
.contains("itemStatuses must be a string of S, F and -"),
"{statuses}"
);
}
let record = parse_replay_children_payload(Some(r#"{"itemStatuses":"S"}"#))
.expect("parses")
.expect("a JS summary is a decision");
let err = ReplayPlan::from_record(&record, 1).expect_err("no reason");
assert!(crate::error::chain_string(&err).contains("unknown completionReason"));
let record = parse_replay_children_payload(Some(
r#"{"totalCount":2,"completionReason":"ALL_COMPLETED","completedIndexes":[0]}"#,
))
.expect("a decision record parses")
.expect("a decision record is a decision");
assert_eq!(record.completed_indexes, Some(vec![0]));
let record = parse_replay_children_payload(Some(
r#"{"totalCount":2,"completionReason":"ALL_COMPLETED","startedIndexes":[1]}"#,
))
.expect("a decision record parses")
.expect("a decision record is a decision");
assert_eq!(record.started_indexes, Some(vec![1]));
for corrupt in [None, Some(""), Some(" ")] {
let err = parse_replay_children_payload(corrupt)
.expect_err("an absent or empty payload is corrupt");
assert!(
crate::error::chain_string(&err).contains("malformed batch decision record"),
"{corrupt:?}"
);
}
for result in [
"not json",
r#"{"unrelated":true}"#,
r#"{"totalCount":2,"completionReason":"ALL_COMPLETED"}"#,
r#"{"startedIndexes":[]}"#,
"[1]",
r#"{"all":[],"completionReason":"ALL_COMPLETED"}"#,
] {
let parsed = parse_replay_children_payload(Some(result)).expect("not corrupt");
assert!(parsed.is_none(), "{result}");
}
}
#[tokio::test]
async fn item_serdes_called_once_per_item() {
let calls = CallLog::default();
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let out: Vec<u32> = ctx
.map(
vec![10_u32, 20],
|_child, item, _idx| async move { Ok(item) },
)
.serdes(calls.clone())
.await
.expect("map succeeds");
assert_eq!(out, vec![10, 20]);
let (c0, c1) = (wire_of("1-1"), wire_of("1-2"));
let mut live = vec![
("deserialize", c0.clone()),
("deserialize", c1.clone()),
("serialize", c0.clone()),
("serialize", c1.clone()),
];
live.sort();
assert_eq!(calls.calls(), live);
let replay_calls = CallLog::default();
let replayed: Vec<u32> = test_ctx(terminal_batch_log(&parent_payload(&client)))
.map(vec![10_u32, 20], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.serdes(replay_calls.clone())
.await
.expect("replay succeeds");
assert_eq!(replayed, vec![10, 20]);
let mut replay = vec![("deserialize", c0), ("deserialize", c1)];
replay.sort();
assert_eq!(replay_calls.calls(), replay);
assert!(
calls
.calls()
.iter()
.chain(replay_calls.calls().iter())
.all(|(_, id)| !id.contains("/item-"))
);
}
#[tokio::test]
async fn batch_item_name_survives_summary_replay() {
use crate::future::Branch;
let branches = || {
vec![
Branch::new("a", |_c| async { Err::<u32, BoxError>("a failed".into()) }),
Branch::unnamed(|_c| async { Err::<u32, BoxError>("b failed".into()) }),
]
};
let tolerate = || {
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_percentage(
100.0,
)
.expect("a valid percentage")
};
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
ctx.parallel(branches())
.completion(tolerate())
.await_batch()
.await
.expect("tolerated failures are a batch result");
let replayed = test_ctx(terminal_batch_log(&parent_payload(&client)))
.parallel(vec![
Branch::<u32>::new("a", |_c| async { unreachable!("a terminal batch replays") }),
Branch::<u32>::unnamed(|_c| async { unreachable!("a terminal batch replays") }),
])
.completion(tolerate())
.await_batch()
.await
.expect("replay succeeds");
let names: Vec<&str> = replayed.items.iter().map(|i| i.name.as_str()).collect();
assert_eq!(names, ["a", "parallel-branch-1"]);
let error_names: Vec<&str> = replayed.errors().map(|e| e.name).collect();
assert_eq!(error_names, names);
}
#[tokio::test]
async fn flat_map_values_replay_from_the_record() {
let (ctx, client) = test_ctx_with_client(CheckpointLog::empty());
let live: Vec<u32> = ctx
.map(
vec![1_u32, 2],
|_child, item, _idx| async move { Ok(item * 10) },
)
.nesting(NestingMode::Flat)
.await
.expect("map succeeds");
assert_eq!(live, vec![10, 20]);
let payload = parent_payload(&client);
let replayed: Vec<u32> = test_ctx(terminal_batch_log(&payload))
.map(vec![1_u32, 2], |_child, _item, _idx| async move {
unreachable!("a terminal batch replays")
})
.nesting(NestingMode::Flat)
.await
.expect("replay succeeds");
assert_eq!(replayed, live);
}
#[tokio::test]
async fn resumed_terminal_item_writes_its_recorded_value() {
let log = CheckpointLog::from_records(vec![
(
wire_of("1"),
CheckpointRecord {
status: CheckpointStatus::Started,
result: None,
..succeeded_record("1", "").1
},
),
succeeded_record("1-1", "\"hello\""),
]);
let (ctx, client) = test_ctx_with_client(log);
let out: Vec<String> = ctx
.map(vec![0_u32], |_child, _item, _idx| async move {
unreachable!("a terminal child replays")
})
.await
.expect("map succeeds");
assert_eq!(out, vec!["hello"]);
assert_eq!(parent_payload_json(&client)["all"][0]["result"], "hello");
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod coordinator_tests {
use crate::DurableContext;
use crate::builders::map_parallel::CompletionConfig;
use crate::driver::InvocationOutcome;
use crate::driver::test_support::{
ScriptedClient, drive_handler, named, operation_like, scripted_ctx, wire_err,
};
use aws_sdk_lambda::types::{OperationAction, OperationStatus};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[derive(Default)]
struct InFlight {
now: AtomicUsize,
peak: AtomicUsize,
events: Mutex<Vec<String>>,
}
impl InFlight {
fn enter(&self, idx: usize) {
let now = self.now.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(now, Ordering::SeqCst);
self.events
.lock()
.expect("lock")
.push(format!("start{idx}"));
}
fn leave(&self, idx: usize) {
self.events.lock().expect("lock").push(format!("end{idx}"));
self.now.fetch_sub(1, Ordering::SeqCst);
}
fn position(&self, event: &str) -> usize {
self.events
.lock()
.expect("lock")
.iter()
.position(|e| e == event)
.unwrap_or_else(|| panic!("no event {event}"))
}
}
#[tokio::test(start_paused = true)]
async fn waiting_item_keeps_slot_and_continues() {
let client = Arc::new(ScriptedClient::new(|updates, history| {
if named(updates, "s1", &OperationAction::Succeed).is_none() {
return Vec::new();
}
named(history, "w0", &OperationAction::Start)
.map(|start| {
operation_like(start, OperationStatus::Succeeded)
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
}));
let ctx = scripted_ctx(&client);
let flight = Arc::new(InFlight::default());
let body_flight = Arc::clone(&flight);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let batch = ctx
.map(
vec![0_u32, 1, 2],
move |child: DurableContext, _item, idx| {
let flight = Arc::clone(&body_flight);
let client = Arc::clone(&body_client);
async move {
flight.enter(idx);
match idx {
0 => {
child.wait(Duration::from_secs(1)).name("w0").await?;
child.step(|_| async { Ok(0_u32) }).name("after0").await?;
}
1 => {
child
.step(move |_| async move {
for _ in 0..1000 {
if client.received("w0", &OperationAction::Start) {
return Ok(1_u32);
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
Err("w0 never started".into())
})
.name("s1")
.await?;
}
_ => {
child.step(|_| async { Ok(2_u32) }).name("s2").await?;
}
}
flight.leave(idx);
Ok(idx)
}
},
)
.max_concurrency(2)
.await_batch()
.await
.map_err(wire_err)?;
let statuses: Vec<String> = batch
.items
.iter()
.map(|item| format!("{:?}", item.status))
.collect();
Ok(statuses.join(","))
})
.await
.expect("no panic");
assert_eq!(
outcome,
InvocationOutcome::Complete("Succeeded,Succeeded,Succeeded".to_owned()),
"the batch completes in one invocation"
);
assert_eq!(flight.peak.load(Ordering::SeqCst), 2);
assert!(
flight.position("end1") < flight.position("start2"),
"item 2 starts after item 1 finished"
);
}
#[tokio::test(start_paused = true)]
async fn item_panic_while_others_wait_propagates() {
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let outcome = drive_handler(ctx, None, |ctx| async move {
let _ = ctx
.map(
vec![0_u32, 1, 2],
|child: DurableContext, _item, idx| async move {
if idx == 0 {
child
.step(|_| async {
tokio::time::sleep(Duration::from_millis(10)).await;
Ok(0_u32)
})
.await?;
panic!("boom while others wait");
}
child.wait(Duration::from_secs(3600)).await?;
Ok(idx)
},
)
.max_concurrency(3)
.await;
Ok("done".to_owned())
})
.await;
let Err(payload) = outcome else {
panic!("expected the item panic, got {outcome:?}");
};
assert_eq!(
payload.downcast_ref::<&str>(),
Some(&"boom while others wait")
);
}
#[tokio::test(start_paused = true)]
async fn early_completion_cancels_running_item_while_others_wait() {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let batch = ctx
.map(
vec![0_u32, 1, 2, 3],
|child: DurableContext, _item, idx| async move {
match idx {
0 => {}
1 | 2 => child.wait(Duration::from_secs(3600)).await?,
_ => {
child
.step(|_| async {
tokio::time::sleep(Duration::from_millis(800)).await;
Ok(())
})
.name("a")
.await?;
child.step(|_| async { Ok(()) }).name("late").await?;
}
}
Ok(idx)
},
)
.max_concurrency(4)
.completion(CompletionConfig::with_min_successful(1))
.await_batch()
.await
.map_err(wire_err)?;
ctx.hold_activity(ctx.coordinator().tasks().wait_idle())
.await;
let waits = ctx.coordinator().live_waits();
let statuses: Vec<String> = batch
.items
.iter()
.map(|item| format!("{:?}", item.status))
.collect();
Ok(format!("{}:{waits}", statuses.join(",")))
})
.await
.expect("no panic");
assert_eq!(
outcome,
InvocationOutcome::Complete("Succeeded,Started,Started,Started:0".to_owned())
);
assert!(
client.updates().iter().all(|u| u.name() != Some("late")),
"the cancelled item wrote no later update"
);
assert_eq!(coordinator.tasks().live(), 0);
}
}
#[cfg(all(test, feature = "test-util"))]
mod batch_terminal_tests {
use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use super::{BatchItemStatus, BatchResult, CompletionReason, MAX_CONCURRENCY_ZERO_MESSAGE};
use crate::builders::map_parallel::CompletionConfig;
use crate::test_util::{ExecutionStatus, LocalRunner};
use crate::{BoxError, Branch, DurableContext};
type Record = (
Vec<(usize, String, BatchItemStatus, Option<i32>)>,
CompletionReason,
);
fn record_of(batch: &BatchResult<i32>) -> Record {
let items = batch
.items
.iter()
.map(|item| (item.index, item.name.clone(), item.status, item.result))
.collect();
(items, batch.reason)
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
type ItemFuture = std::pin::Pin<Box<dyn Future<Output = Result<i32, BoxError>> + Send>>;
fn logged_map(
ctx: &DurableContext,
invocation: usize,
log: &Arc<Mutex<Vec<usize>>>,
) -> crate::builders::MapBuilder<
i32,
i32,
impl Fn(DurableContext, i32, usize) -> ItemFuture + Send + Sync + 'static,
ItemFuture,
> {
let log = Arc::clone(log);
ctx.map(vec![1, 2], move |child, i: i32, _idx| -> ItemFuture {
let log = Arc::clone(&log);
Box::pin(async move {
child
.step(move |_| async move {
lock(&log).push(invocation);
Ok(i * 10)
})
.name("scale")
.await
.map_err(Into::into)
})
})
.name("m")
}
#[tokio::test]
async fn spawn_batch_runs_while_the_handler_waits() {
let invocations = Arc::new(AtomicUsize::new(0));
let log = Arc::new(Mutex::new(Vec::new()));
let (inv, step_log) = (Arc::clone(&invocations), Arc::clone(&log));
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let invocation = inv.fetch_add(1, Ordering::SeqCst) + 1;
let log = Arc::clone(&step_log);
async move {
let batch = logged_map(&ctx, invocation, &log).spawn_batch();
ctx.wait(Duration::from_secs(1)).await?;
let batch = batch.await?;
Ok::<_, BoxError>(batch.results().copied().collect::<Vec<_>>())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Succeeded);
assert_eq!(result.output(), Some(&vec![10, 20]));
assert_eq!(result.invocation_count(), 2);
assert_eq!(*lock(&log), [1, 1]);
}
#[tokio::test]
async fn spawn_batch_equals_await_batch() {
let spawned: Arc<Mutex<Vec<Record>>> = Arc::default();
let awaited: Arc<Mutex<Vec<Record>>> = Arc::default();
let records = Arc::clone(&spawned);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let records = Arc::clone(&records);
async move {
let log = Arc::default();
let batch = logged_map(&ctx, 0, &log).spawn_batch();
ctx.wait(Duration::from_secs(1)).await?;
let batch = batch.await?;
lock(&records).push(record_of(&batch));
Ok::<_, BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Succeeded);
let records = Arc::clone(&awaited);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let records = Arc::clone(&records);
async move {
let log = Arc::default();
let batch = logged_map(&ctx, 0, &log).await_batch().await?;
lock(&records).push(record_of(&batch));
ctx.wait(Duration::from_secs(1)).await?;
Ok::<_, BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Succeeded);
let spawned = lock(&spawned).clone();
let awaited = lock(&awaited).clone();
let [spawned] = spawned.as_slice() else {
unreachable!("A awaits the batch once, after the wait: {spawned:?}");
};
assert_eq!(awaited.len(), 2, "B awaits the batch in both invocations");
for record in &awaited {
assert_eq!(record, spawned);
}
}
#[tokio::test]
async fn parallel_tolerated_failure_percentage_fails_the_batch() {
let seen: Arc<Mutex<Option<Record>>> = Arc::default();
let slot = Arc::clone(&seen);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let slot = Arc::clone(&slot);
async move {
let branches = vec![
Branch::new("ok", |_| async { Ok(1) }),
Branch::new("bad-1", |_| async { Err::<i32, BoxError>("boom".into()) }),
Branch::new("bad-2", |_| async { Err::<i32, BoxError>("boom".into()) }),
];
let batch = ctx
.parallel(branches)
.max_concurrency(1)
.completion(
CompletionConfig::with_tolerated_failure_percentage(50.0)
.expect("a valid percentage"),
)
.await_batch()
.await?;
*lock(&slot) = Some(record_of(&batch));
Ok::<_, BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Succeeded);
let (items, reason) = lock(&seen).clone().expect("the handler recorded the batch");
let statuses: Vec<_> = items
.iter()
.map(|(index, _, status, value)| (*index, *status, *value))
.collect();
assert_eq!(
statuses,
[
(0, BatchItemStatus::Succeeded, Some(1)),
(1, BatchItemStatus::Failed, None),
(2, BatchItemStatus::Failed, None),
]
);
assert_eq!(reason, CompletionReason::FailureToleranceExceeded);
}
async fn replay_identity_case(
use_await_batch: bool,
) -> (Option<String>, Option<String>, usize) {
let invocations = Arc::new(AtomicUsize::new(0));
let bodies = Arc::new(AtomicUsize::new(0));
let ran = Arc::clone(&bodies);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let first = invocations.fetch_add(1, Ordering::SeqCst) == 0;
let ran = Arc::clone(&ran);
async move {
if first {
ctx.step(|_| async { Ok(1_i32) }).name("s").await?;
ctx.wait(Duration::from_secs(1)).await?;
return Ok::<_, BoxError>(());
}
let map = ctx
.map(vec![1], move |_child, i: i32, _idx| {
let ran = Arc::clone(&ran);
async move {
ran.fetch_add(1, Ordering::SeqCst);
Ok(i)
}
})
.name("m");
if use_await_batch {
map.await_batch().await?;
} else {
map.await?;
}
Ok(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Failed);
(
result.error_type().map(str::to_owned),
result.error_message().map(str::to_owned),
bodies.load(Ordering::SeqCst),
)
}
#[tokio::test]
async fn await_batch_checks_replay_identity_like_await() {
let batch = replay_identity_case(true).await;
let plain = replay_identity_case(false).await;
assert_eq!(batch.0.as_deref(), Some("NonDeterministicExecutionError"));
assert_eq!(plain.0.as_deref(), Some("NonDeterministicExecutionError"));
assert!(batch.1.is_some());
assert_eq!(batch.1, plain.1);
assert_eq!((batch.2, plain.2), (0, 0), "no item body ran");
}
#[tokio::test]
async fn parallel_spawn_batch_returns_branch_statuses() {
let seen: Arc<Mutex<Option<Record>>> = Arc::default();
let slot = Arc::clone(&seen);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let slot = Arc::clone(&slot);
async move {
let branches = vec![
Branch::new("ok", |_| async { Ok(1) }),
Branch::new("bad", |_| async { Err::<i32, BoxError>("boom".into()) }),
];
let batch = ctx
.parallel(branches)
.completion(CompletionConfig::with_tolerated_failure_count(1))
.spawn_batch();
ctx.wait(Duration::from_secs(1)).await?;
let batch = batch.await?;
*lock(&slot) = Some(record_of(&batch));
Ok::<_, BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Succeeded);
let (items, reason) = lock(&seen).clone().expect("the handler recorded the batch");
let statuses: Vec<_> = items
.iter()
.map(|(index, _, status, value)| (*index, *status, *value))
.collect();
assert_eq!(
statuses,
[
(0, BatchItemStatus::Succeeded, Some(1)),
(1, BatchItemStatus::Failed, None),
]
);
assert_eq!(reason, CompletionReason::AllCompleted);
}
#[derive(Clone, Copy)]
enum Terminal {
Await,
AwaitBatch,
SpawnBatch,
}
async fn max_concurrency_zero_case(terminal: Terminal) {
let after = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&after);
let result = LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let counter = Arc::clone(&counter);
async move {
let map = ctx
.map(vec![1, 2], |_child, i: i32, _idx| async move { Ok(i) })
.name("m")
.max_concurrency(0);
match terminal {
Terminal::Await => drop(map.await),
Terminal::AwaitBatch => drop(map.await_batch().await),
Terminal::SpawnBatch => drop(map.spawn_batch().await),
}
counter.fetch_add(1, Ordering::SeqCst);
Ok::<_, BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Failed);
assert_eq!(result.error_type(), Some("Error"));
assert_eq!(result.error_message(), Some(MAX_CONCURRENCY_ZERO_MESSAGE));
assert_eq!(
MAX_CONCURRENCY_ZERO_MESSAGE,
"Invalid max_concurrency: 0. Must be a positive number or unset for unlimited \
concurrency."
);
assert!(
result.operation("m").is_none(),
"no batch record is written"
);
assert_eq!(after.load(Ordering::SeqCst), 0, "the await never resolves");
}
#[tokio::test]
async fn max_concurrency_zero_fails_the_execution() {
max_concurrency_zero_case(Terminal::Await).await;
max_concurrency_zero_case(Terminal::AwaitBatch).await;
max_concurrency_zero_case(Terminal::SpawnBatch).await;
}
#[tokio::test]
async fn parallel_max_concurrency_zero_fails_the_execution() {
let result = LocalRunner::new()
.run(
|(): (), ctx: DurableContext| async move {
let branches = vec![Branch::new("ok", |_| async { Ok(1) })];
drop(ctx.parallel(branches).name("p").max_concurrency(0).await);
Ok::<_, BoxError>(())
},
(),
)
.await
.expect("the run finishes");
assert_eq!(result.status(), ExecutionStatus::Failed);
assert_eq!(result.error_message(), Some(MAX_CONCURRENCY_ZERO_MESSAGE));
assert!(result.operation("p").is_none());
}
}