#![cfg(feature = "test-util")]
#![expect(clippy::expect_used)]
use std::sync::{Arc, Mutex};
use std::time::Duration;
use aws_durable_execution_sdk as durable;
use durable::builders::map_parallel::{CompletionConfig, CompletionDecision, CompletionOutcome};
use durable::test_util::LocalRunner;
type Seen = Arc<Mutex<Vec<Vec<String>>>>;
async fn run_parked_item_batch(size: usize) -> Vec<Vec<String>> {
let seen: Seen = Arc::new(Mutex::new(Vec::new()));
let handler_seen = Arc::clone(&seen);
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let seen = Arc::clone(&handler_seen);
async move {
let batch = ctx
.map(vec![0_u32, 1, 2], move |child, item, _idx| async move {
if item == 2 {
child.wait(Duration::from_secs(5)).name("park").await?;
}
let value = child
.step(move |_| async move { Ok("x".repeat(size)) })
.name("value")
.await?;
Ok(value)
})
.name("m")
.completion(CompletionConfig::with_min_successful(2))
.await_batch()
.await?;
let statuses = batch
.items
.iter()
.map(|item| format!("{:?}", item.status))
.collect();
seen.lock().expect("lock").push(statuses);
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "the execution succeeds: {result}");
seen.lock().expect("lock").clone()
}
const PARKED_STARTED: [&str; 3] = ["Succeeded", "Succeeded", "Started"];
#[tokio::test]
async fn parked_item_is_started_live_and_on_replay_small() {
assert_eq!(
run_parked_item_batch(10).await,
[PARKED_STARTED, PARKED_STARTED],
"the live invocation and the replay report the parked item as Started"
);
}
#[tokio::test]
async fn parked_item_is_started_live_and_on_replay_oversized() {
assert_eq!(
run_parked_item_batch(150 * 1024).await,
[PARKED_STARTED, PARKED_STARTED],
"the live invocation and the replay report the parked item as Started"
);
}
type Outcomes = Arc<Mutex<Vec<(String, String)>>>;
async fn run_custom_completion(size: usize, outcome: CompletionOutcome) -> Vec<(String, String)> {
let seen: Outcomes = Arc::new(Mutex::new(Vec::new()));
let handler_seen = Arc::clone(&seen);
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let seen = Arc::clone(&handler_seen);
async move {
let batch = ctx
.map(vec![0_u32, 1, 2], move |child, _item, _idx| async move {
let value = child
.step(move |_| async move { Ok("x".repeat(size)) })
.name("value")
.await?;
Ok(value)
})
.name("m")
.max_concurrency(1)
.completion(CompletionConfig::with_completion_predicate(move |stats| {
if stats.settled() >= 2 {
CompletionDecision::complete(outcome)
} else {
CompletionDecision::Continue
}
}))
.await_batch()
.await?;
seen.lock().expect("lock").push((
batch.reason.as_str().to_owned(),
format!("{:?}", batch.status()),
));
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "the execution succeeds: {result}");
seen.lock().expect("lock").clone()
}
fn twice(reason: &str, status: &str) -> [(String, String); 2] {
let entry = (reason.to_owned(), status.to_owned());
[entry.clone(), entry]
}
#[tokio::test]
async fn custom_succeeded_survives_small_replay() {
assert_eq!(
run_custom_completion(10, CompletionOutcome::Succeeded).await,
twice("CUSTOM_COMPLETION_SUCCEEDED", "Succeeded"),
"the live invocation and the replay record the same reason"
);
}
#[tokio::test]
async fn custom_failed_survives_small_replay() {
assert_eq!(
run_custom_completion(10, CompletionOutcome::Failed).await,
twice("CUSTOM_COMPLETION_FAILED", "Failed"),
"the live invocation and the replay record the same reason"
);
}
#[tokio::test]
async fn custom_failed_survives_oversized_replay() {
assert_eq!(
run_custom_completion(150 * 1024, CompletionOutcome::Failed).await,
twice("CUSTOM_COMPLETION_FAILED", "Failed"),
"the live invocation and the replay record the same reason"
);
}
type Calls = Arc<Mutex<Vec<(String, usize)>>>;
async fn run_counted_predicate(
nesting: durable::builders::map_parallel::NestingMode,
size: usize,
) -> Vec<(String, usize)> {
let seen: Calls = Arc::new(Mutex::new(Vec::new()));
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let handler_seen = Arc::clone(&seen);
let handler_calls = Arc::clone(&calls);
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let seen = Arc::clone(&handler_seen);
let calls = Arc::clone(&handler_calls);
let predicate_calls = Arc::clone(&calls);
async move {
let batch = ctx
.map(vec![0_u32, 1, 2], move |child, _item, _idx| async move {
let first = child
.step(move |_| async move { Ok("x".repeat(size)) })
.name("first")
.await?;
let second = child
.step(move |_| async move { Ok("y".repeat(size)) })
.name("second")
.await?;
Ok(first + &second)
})
.name("m")
.max_concurrency(1)
.nesting(nesting)
.completion(CompletionConfig::with_completion_predicate(move |stats| {
predicate_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if stats.settled() >= 2 {
CompletionDecision::complete(CompletionOutcome::Failed)
} else {
CompletionDecision::Continue
}
}))
.await_batch()
.await?;
seen.lock().expect("lock").push((
batch.reason.as_str().to_owned(),
calls.load(std::sync::atomic::Ordering::SeqCst),
));
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "the execution succeeds: {result}");
seen.lock().expect("lock").clone()
}
fn assert_rebuild_skips_predicate(seen: &[(String, usize)]) {
assert_eq!(seen.len(), 2, "two invocations: {seen:?}");
let (live_reason, live_calls) = seen.first().expect("live invocation");
let (replay_reason, replay_calls) = seen.get(1).expect("replay invocation");
assert_eq!(live_reason, "CUSTOM_COMPLETION_FAILED");
assert_eq!(
replay_reason, live_reason,
"the replay reads the recorded reason"
);
assert!(*live_calls > 0, "the live run asks the predicate");
assert_eq!(
replay_calls, live_calls,
"rebuilding the completed batch calls the predicate no more"
);
}
#[tokio::test]
async fn flat_oversized_rebuild_does_not_call_predicate() {
let seen = run_counted_predicate(
durable::builders::map_parallel::NestingMode::Flat,
80 * 1024,
)
.await;
assert_rebuild_skips_predicate(&seen);
}
#[tokio::test]
async fn oversized_child_rebuild_does_not_call_predicate() {
let seen = run_counted_predicate(
durable::builders::map_parallel::NestingMode::Normal,
150 * 1024,
)
.await;
assert_rebuild_skips_predicate(&seen);
}
#[tokio::test]
async fn invalid_percentage_starts_no_batch() {
let errors: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
let handler_errors = Arc::clone(&errors);
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let errors = Arc::clone(&handler_errors);
async move {
let attempts = [
(
"invalid-constructor",
CompletionConfig::with_tolerated_failure_percentage(150.0),
),
(
"invalid-builder",
CompletionConfig::builder()
.tolerated_failure_percentage(150.0)
.build(),
),
];
for (name, built) in attempts {
match built {
Ok(config) => {
ctx.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name(name)
.completion(config)
.await?;
}
Err(err) => errors.lock().expect("lock").push(err.to_string()),
}
}
ctx.map(vec![1_u32], |_child, item, _idx| async move { Ok(item) })
.name("ok")
.completion(CompletionConfig::with_tolerated_failure_percentage(100.0)?)
.await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "the execution succeeds: {result}");
assert_eq!(result.invocation_count(), 1);
let expected =
"invalid CompletionConfig: tolerated_failure_percentage must be 0-100, got 150".to_owned();
assert_eq!(
*errors.lock().expect("lock"),
[expected.clone(), expected],
"both construction paths return the same error"
);
assert!(result.operation("invalid-constructor").is_none());
assert!(result.operation("invalid-builder").is_none());
assert_eq!(
result.operation("ok").expect("the valid map ran").status(),
durable::OperationStatus::Succeeded
);
assert!(
result
.operations()
.iter()
.all(|op| op.status() != durable::OperationStatus::Started),
"no operation stays STARTED: {result}"
);
}
struct PaddedSerdes;
const PAD: usize = 300 * 1024;
impl durable::Serdes<String> for PaddedSerdes {
fn serialize(
&self,
value: String,
_context: durable::serdes::SerdesContext,
) -> impl Future<Output = Result<String, durable::BoxError>> + Send {
std::future::ready(Ok(format!("{}{value}", " ".repeat(PAD))))
}
fn deserialize(
&self,
wire: String,
_context: durable::serdes::SerdesContext,
) -> impl Future<Output = Result<String, durable::BoxError>> + Send {
std::future::ready(
wire.get(PAD..)
.map(str::to_owned)
.ok_or_else(|| "the payload has no padding".into()),
)
}
}
#[tokio::test]
async fn empty_map_with_oversized_result_replays_after_a_wait() {
let result = LocalRunner::new()
.run(
|_event: serde_json::Value, ctx: durable::DurableContext| async move {
let values = ctx
.map(Vec::<u32>::new(), |_, item, _| async move { Ok(item) })
.name("b")
.result_serdes(PaddedSerdes)
.await?;
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(values.len())
},
serde_json::Value::Null,
)
.await
.expect("runner");
let op = result.operation("b").expect("record b");
assert!(
matches!(
op.details(),
durable::test_util::TestOperationDetails::Context(c) if c.replay_children()
),
"the record is the oversized summary"
);
assert!(result.is_success(), "{result} {:?}", result.error_message());
assert_eq!(result.output(), Some(&0));
}
async fn completed_before_first_dispatch(
nesting: durable::builders::map_parallel::NestingMode,
) -> Vec<(usize, &'static str)> {
let seen: Arc<Mutex<Vec<(usize, &'static str)>>> = Arc::default();
let handler_seen = Arc::clone(&seen);
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let seen = Arc::clone(&handler_seen);
async move {
let batch = ctx
.map(vec![0_u32, 1, 2], |_, item, _| async move { Ok(item) })
.name("b")
.nesting(nesting)
.completion(CompletionConfig::with_completion_predicate(|_| {
CompletionDecision::complete(CompletionOutcome::Succeeded)
}))
.result_serdes(PaddedSerdes)
.await_batch()
.await?;
let seen_now = (batch.items.len(), batch.reason.as_str());
seen.lock().expect("lock").push(seen_now);
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "{result} {:?}", result.error_message());
std::mem::take(&mut *seen.lock().expect("lock"))
}
#[tokio::test]
async fn map_completed_before_first_dispatch_with_oversized_result_replays() {
let nesting = durable::builders::map_parallel::NestingMode::Normal;
assert_eq!(
completed_before_first_dispatch(nesting).await,
vec![(0, "CUSTOM_COMPLETION_SUCCEEDED"); 2]
);
}
#[tokio::test]
async fn flat_map_completed_before_first_dispatch_with_oversized_result_replays() {
let nesting = durable::builders::map_parallel::NestingMode::Flat;
assert_eq!(
completed_before_first_dispatch(nesting).await,
vec![(0, "CUSTOM_COMPLETION_SUCCEEDED"); 2]
);
}
#[tokio::test]
async fn empty_batch_replay_uses_recorded_reason_without_reinvoking_predicate() {
for nesting in [
durable::builders::map_parallel::NestingMode::Normal,
durable::builders::map_parallel::NestingMode::Flat,
] {
let seen: Calls = Arc::new(Mutex::new(Vec::new()));
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let (handler_seen, handler_calls) = (Arc::clone(&seen), Arc::clone(&calls));
let result = LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let seen = Arc::clone(&handler_seen);
let calls = Arc::clone(&handler_calls);
let predicate_calls = Arc::clone(&calls);
async move {
let batch = ctx
.map(Vec::<u32>::new(), |_, item, _| async move { Ok(item) })
.name("b")
.nesting(nesting)
.completion(CompletionConfig::with_completion_predicate(move |_| {
predicate_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
CompletionDecision::complete(CompletionOutcome::Succeeded)
}))
.result_serdes(PaddedSerdes)
.await_batch()
.await?;
seen.lock().expect("lock").push((
batch.reason.as_str().to_owned(),
calls.load(std::sync::atomic::Ordering::SeqCst),
));
ctx.wait(Duration::from_secs(1)).name("after").await?;
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert!(result.is_success(), "{result} {:?}", result.error_message());
assert_eq!(
*seen.lock().expect("lock"),
vec![("CUSTOM_COMPLETION_SUCCEEDED".to_owned(), 1); 2],
"{nesting:?}: the replay reads the recorded reason and asks nothing"
);
}
}
async fn resumed_batch(completion: CompletionConfig, values: bool) -> Vec<(usize, String)> {
let seen = Arc::new(Mutex::new(Vec::new()));
let calls = Arc::new(Mutex::new(0_usize));
let (handler_seen, handler_calls) = (Arc::clone(&seen), Arc::clone(&calls));
let result = LocalRunner::new()
.max_invocations(4)
.fail_checkpoints_after_retryable(2, 1)
.run(
move |_event: serde_json::Value, ctx: durable::DurableContext| {
let (seen, calls, completion) = (
Arc::clone(&handler_seen),
Arc::clone(&handler_calls),
completion.clone(),
);
async move {
let invocation = {
let mut n = calls.lock().expect("lock");
*n += 1;
*n
};
let failed = Arc::new(tokio::sync::Notify::new());
let map = ctx
.map(vec![0_u32, 1], move |_child, item, _idx| {
let failed = Arc::clone(&failed);
async move {
if item == 1 {
failed.notify_one();
return Err::<u32, durable::BoxError>("item 1 fails".into());
}
failed.notified().await;
Ok(7)
}
})
.max_concurrency(2)
.completion(completion);
let outcome = if values {
match map.await {
Ok(values) => format!("Ok({values:?})"),
Err(err) => {
let wire = err.wire();
let message = wire.and_then(durable::WireError::error_message);
format!("Err({})", message.unwrap_or_default())
}
}
} else {
map.await_batch().await?.reason.as_str().to_owned()
};
seen.lock().expect("lock").push((invocation, outcome));
Ok::<_, durable::BoxError>(())
}
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert_eq!(
result.invocation_count(),
2,
"the batch outcome write failed once"
);
let items: Vec<_> = ["map-item-0", "map-item-1"]
.iter()
.map(|name| result.operation(name).expect("the item record").status())
.collect();
assert_eq!(
items,
[
durable::OperationStatus::Succeeded,
durable::OperationStatus::Failed
],
"invocation 1 recorded both item outcomes"
);
std::mem::take(&mut *seen.lock().expect("lock"))
}
fn min_one_no_failure() -> CompletionConfig {
CompletionConfig::builder()
.min_successful(1)
.tolerated_failure_count(0)
.build()
.expect("valid completion config")
}
#[tokio::test]
async fn resumed_batch_checks_the_failure_tolerance_on_every_recorded_outcome() {
assert_eq!(
resumed_batch(min_one_no_failure(), false).await,
[(2, "FAILURE_TOLERANCE_EXCEEDED".to_owned())]
);
}
#[tokio::test]
async fn resumed_batch_await_returns_the_recorded_failure() {
assert_eq!(
resumed_batch(min_one_no_failure(), true).await,
[(2, "Err(item 1 fails)".to_owned())]
);
}
#[tokio::test]
async fn resumed_batch_with_every_outcome_recorded_is_all_completed() {
assert_eq!(
resumed_batch(CompletionConfig::with_min_successful(1), false).await,
[(2, "ALL_COMPLETED".to_owned())]
);
}
#[tokio::test]
async fn concurrent_min_successful_zero_completes_at_first_settle() {
let result = LocalRunner::new()
.run(
|_event: serde_json::Value, ctx: durable::DurableContext| async move {
let batch = ctx
.map(vec![0_u32, 1], |_child, item, _idx| async move {
if item == 1 {
std::future::pending::<()>().await;
}
Ok(item)
})
.max_concurrency(2)
.completion(CompletionConfig::with_min_successful(0))
.await_batch()
.await?;
let statuses: Vec<String> = batch
.items
.iter()
.map(|i| format!("{:?}", i.status))
.collect();
Ok::<_, durable::BoxError>((batch.reason.as_str().to_owned(), statuses))
},
serde_json::Value::Null,
)
.await
.expect("runner");
assert_eq!(
result.output().expect("output"),
&(
"MIN_SUCCESSFUL_REACHED".to_owned(),
vec!["Succeeded".to_owned(), "Started".to_owned()]
)
);
}