use aws_sdk_lambda::types::OperationUpdate;
use crate::context::{DurableContext, SubmittedWrite};
use crate::engine::{CheckpointStatus, CheckpointStatusView, OperationId};
use crate::error::OperationError;
use crate::plugin::InvocationPlugins;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum StartOrder {
QueueThenHook,
AwaitResponse,
AwaitUrgentResponse,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Cycle {
Start { is_replay: bool },
Resume,
Record,
}
pub(crate) struct StartSpec<'a> {
pub(crate) op_id: &'a OperationId,
pub(crate) operation_type: &'static str,
pub(crate) sub_type: &'a str,
pub(crate) name: Option<&'a str>,
pub(crate) order: StartOrder,
}
pub(crate) enum StartStep {
Refused,
Recorded(CheckpointStatusView),
Queued(SubmittedWrite),
}
pub(crate) fn rebuild_holds(
ctx: &DurableContext,
operation_type: &str,
status: Option<&CheckpointStatus>,
) -> bool {
if !ctx.is_rebuilding() {
return false;
}
let Some(status) = status else {
return true;
};
match operation_type {
"Callback" => *status == CheckpointStatus::Ready,
"ChainedInvoke" => matches!(
status,
CheckpointStatus::Started | CheckpointStatus::Pending | CheckpointStatus::Ready
),
_ => !matches!(
status,
CheckpointStatus::Succeeded | CheckpointStatus::Failed
),
}
}
pub(crate) fn start_step(
ctx: &DurableContext,
spec: &StartSpec<'_>,
classify: impl FnOnce(&CheckpointStatusView) -> Cycle,
start_update: impl FnOnce() -> OperationUpdate,
start_hook: impl FnOnce(&InvocationPlugins, bool),
) -> Result<StartStep, OperationError> {
if !ctx.admission().admits(ctx.owners()) {
return Ok(StartStep::Refused);
}
ctx.enforce_task_ownership()?;
let view = ctx.checkpoint_view_validated(
spec.op_id.positional(),
spec.op_id.wire(),
spec.operation_type,
Some(spec.sub_type),
spec.name,
)?;
if rebuild_holds(
ctx,
spec.operation_type,
view.as_ref().map(|view| &view.status),
) {
return Ok(StartStep::Refused);
}
let is_replay = match view {
None => false,
Some(view) => match classify(&view) {
Cycle::Start { is_replay } => is_replay,
Cycle::Resume => {
if let Some(plugins) = ctx.plugins() {
start_hook(plugins, true);
}
return Ok(StartStep::Recorded(view));
}
Cycle::Record => return Ok(StartStep::Recorded(view)),
},
};
let update = start_update();
let write = match spec.order {
StartOrder::QueueThenHook => {
let reserved = ctx.reserve_checkpoint_updates(vec![update]);
if let Some(plugins) = ctx.plugins() {
start_hook(plugins, is_replay);
}
reserved.release()
}
StartOrder::AwaitResponse => ctx.submit_checkpoint_updates(vec![update]),
StartOrder::AwaitUrgentResponse => ctx.submit_checkpoint_updates_urgent(vec![update]),
};
Ok(StartStep::Queued(write))
}
#[cfg(test)]
#[expect(clippy::panic, clippy::indexing_slicing)] mod tests {
use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
use super::{Cycle, StartOrder, StartSpec, StartStep, start_step};
use crate::client::{
CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput,
InMemoryExecutionClient,
};
use crate::context::DurableContext;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus, OperationId};
use crate::error::OperationError;
use crate::plugin::test_support::OpHooks;
use crate::step::{StepSemantics, step_cycle};
fn step_record(
status: CheckpointStatus,
attempt: u32,
op_type: &str,
) -> (String, CheckpointRecord) {
let id = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: id.clone(),
status,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: Some(op_type.to_owned()),
sub_type: Some("Step".to_owned()),
due_at: None,
op_name: Some("a".to_owned()),
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
(id, record)
}
fn context(
records: Vec<(String, CheckpointRecord)>,
client: Arc<dyn ExecutionClient>,
window: Duration,
hooks: Option<&OpHooks>,
) -> DurableContext {
DurableContext::new_root_with_client_and_defaults(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::from_records(records)),
client,
"token-0".to_owned(),
window,
0,
hooks.map(OpHooks::plugins),
)
}
fn in_memory() -> Arc<InMemoryExecutionClient> {
Arc::new(InMemoryExecutionClient::new(Vec::new()))
}
#[test]
fn rebuild_holds_matches_rebuilt_body_status_matrix() {
use CheckpointStatus::{
Cancelled, Failed, Pending, Ready, Started, Stopped, Succeeded, TimedOut,
};
let root = DurableContext::test_context();
let rebuilding = root.new_child_rebuilding("1", None, true);
let statuses = [
Started, Pending, Ready, Succeeded, Failed, Cancelled, TimedOut, Stopped,
];
let rows: [(&str, Vec<CheckpointStatus>); 5] = [
("Step", vec![Succeeded, Failed]),
("Wait", vec![Succeeded, Failed]),
("Context", vec![Succeeded, Failed]),
(
"ChainedInvoke",
vec![Succeeded, Failed, Cancelled, TimedOut, Stopped],
),
(
"Callback",
vec![
Started, Pending, Succeeded, Failed, Cancelled, TimedOut, Stopped,
],
),
];
for (operation_type, returns) in &rows {
assert!(
super::rebuild_holds(&rebuilding, operation_type, None),
"{operation_type} without a record"
);
assert!(!super::rebuild_holds(&root, operation_type, None));
for status in &statuses {
assert_eq!(
super::rebuild_holds(&rebuilding, operation_type, Some(status)),
!returns.contains(status),
"{operation_type} {status:?}"
);
assert!(!super::rebuild_holds(&root, operation_type, Some(status)));
}
}
}
fn start_update(op_id: &OperationId, r#type: OperationType, sub_type: &str) -> OperationUpdate {
OperationUpdate::builder()
.id(op_id.wire())
.r#type(r#type)
.sub_type(sub_type)
.name("a")
.action(OperationAction::Start)
.build()
.expect("all required OperationUpdate fields set")
}
fn step_start(
ctx: &DurableContext,
semantics: StepSemantics,
) -> Result<StartStep, OperationError> {
let op_id = ctx.mint_id();
let order = match semantics {
StepSemantics::AtLeastOncePerRetry => StartOrder::QueueThenHook,
StepSemantics::AtMostOncePerRetry => StartOrder::AwaitResponse,
};
start_step(
ctx,
&StartSpec {
op_id: &op_id,
operation_type: "Step",
sub_type: "Step",
name: Some("a"),
order,
},
|view| step_cycle(semantics, view),
|| start_update(&op_id, OperationType::Step, "Step"),
|plugins, is_replay| {
let info = crate::plugin::OperationInfo::for_operation(
ctx,
op_id.wire(),
crate::OperationKind::Step,
"Step",
Some("a"),
)
.with_status(crate::OperationStatus::Started)
.with_replay(is_replay);
crate::plugin::operation_started(plugins, &info);
},
)
}
fn response_then_hook_start(
ctx: &DurableContext,
operation_type: &'static str,
r#type: OperationType,
order: StartOrder,
) -> StartStep {
let op_id = ctx.mint_id();
start_step(
ctx,
&StartSpec {
op_id: &op_id,
operation_type,
sub_type: operation_type,
name: Some("a"),
order,
},
|_| Cycle::Record,
|| start_update(&op_id, r#type, operation_type),
|_, _| unreachable!("no record resumes, and the hook follows the response"),
)
.expect("the start step succeeds")
}
fn starts(hooks: &OpHooks) -> Vec<bool> {
hooks
.records()
.iter()
.filter(|r| r.hook == "start")
.map(|r| r.replay)
.collect()
}
#[tokio::test]
async fn start_step_queues_start_before_returning() {
let client = in_memory();
let ctx = context(Vec::new(), client.clone(), Duration::ZERO, None);
let started = step_start(&ctx, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert!(matches!(started, StartStep::Queued(_)));
assert_eq!(
ctx.queued_update_ids(),
[crate::engine::compute_wire_id_public("1")],
"START is queued before anything is polled"
);
assert!(client.recorded_updates().is_empty(), "nothing is sent yet");
let StartStep::Queued(write) = started else {
unreachable!("checked above")
};
write.await.expect("START is written");
assert_eq!(client.recorded_updates().len(), 1);
}
#[tokio::test]
async fn start_step_returns_record_without_queuing() {
let ctx = context(
vec![step_record(CheckpointStatus::Started, 0, "Step")],
in_memory(),
Duration::ZERO,
None,
);
let started = step_start(&ctx, StepSemantics::AtMostOncePerRetry).expect("no error");
let StartStep::Recorded(view) = started else {
panic!("a STARTED record queues no START");
};
assert_eq!(view.status, CheckpointStatus::Started);
assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
}
#[tokio::test]
async fn start_step_refused_by_admission_queues_nothing() {
let client = in_memory();
let root = context(Vec::new(), client.clone(), Duration::ZERO, None);
let child = root.new_child("1", None);
root.admission()
.abandon([crate::engine::compute_wire_id_public("1").as_str()]);
let started = step_start(&child, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert!(matches!(started, StartStep::Refused));
assert_eq!(child.queued_update_ids(), Vec::<String>::new());
let op_id = child.mint_id();
let step = crate::step::StepExecution {
ctx: child.clone(),
op_id,
name: Some("b".to_owned()),
retry_strategy: None,
serdes: crate::JsonSerdes,
semantics: StepSemantics::AtLeastOncePerRetry,
sub_type: crate::OperationSubType::STEP,
plugin_name: None,
closure: |_| async { Ok(1_u32) },
_marker: std::marker::PhantomData,
};
assert!(
tokio::time::timeout(Duration::from_millis(50), step.execute())
.await
.is_err(),
"a refused operation never resolves"
);
assert_eq!(client.recorded_updates(), Vec::new());
}
#[tokio::test]
async fn start_step_fires_live_hook_only_for_queue_then_hook() {
let hooks = OpHooks::default();
let ctx = context(Vec::new(), in_memory(), Duration::ZERO, Some(&hooks));
let started = step_start(&ctx, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert!(matches!(started, StartStep::Queued(_)));
assert_eq!(
starts(&hooks),
[false],
"a default step fires the live hook"
);
let hooks = OpHooks::default();
let ctx = context(Vec::new(), in_memory(), Duration::ZERO, Some(&hooks));
let started =
response_then_hook_start(&ctx, "Wait", OperationType::Wait, StartOrder::AwaitResponse);
assert!(matches!(started, StartStep::Queued(_)));
assert!(
starts(&hooks).is_empty(),
"a wait fires nothing in the start step"
);
}
#[tokio::test]
async fn start_step_fires_replay_hook_for_started_record() {
let hooks = OpHooks::default();
let ctx = context(
vec![step_record(CheckpointStatus::Started, 0, "Step")],
in_memory(),
Duration::ZERO,
Some(&hooks),
);
let started = step_start(&ctx, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert!(matches!(started, StartStep::Recorded(_)));
assert_eq!(
starts(&hooks),
[true],
"the replay hook fired before the return"
);
assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
}
#[tokio::test]
async fn start_step_queues_callback_start_urgently_before_returning() {
let client = in_memory();
let ctx = context(Vec::new(), client.clone(), Duration::from_hours(1), None);
let started = response_then_hook_start(
&ctx,
"Callback",
OperationType::Callback,
StartOrder::AwaitUrgentResponse,
);
let StartStep::Queued(write) = started else {
panic!("a callback without a record queues START");
};
assert_eq!(
ctx.queued_update_ids(),
[crate::engine::compute_wire_id_public("1")],
"START joined the coalescer"
);
tokio::task::yield_now().await;
assert_eq!(
client.recorded_updates().len(),
1,
"the urgent flush reached the client without the delay"
);
write.await.expect("START is written");
}
#[tokio::test]
async fn start_step_returns_replay_identity_error_without_queuing() {
let ctx = context(
vec![step_record(CheckpointStatus::Started, 0, "Wait")],
in_memory(),
Duration::ZERO,
None,
);
let Err(err) = step_start(&ctx, StepSemantics::AtLeastOncePerRetry) else {
panic!("a record of another type fails the replay identity check");
};
assert!(err.to_string().contains("non_deterministic"), "{err}");
assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
}
#[tokio::test]
async fn start_step_queues_start_for_new_attempt_states() {
for status in [
CheckpointStatus::Ready,
CheckpointStatus::Cancelled,
CheckpointStatus::TimedOut,
CheckpointStatus::Stopped,
] {
let hooks = OpHooks::default();
let client = in_memory();
let ctx = context(
vec![step_record(status.clone(), 1, "Step")],
client.clone(),
Duration::ZERO,
Some(&hooks),
);
let started = step_start(&ctx, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert_eq!(
starts(&hooks),
[true],
"{status:?}: the retry's hook is a replay, fired before the return"
);
assert_eq!(ctx.queued_update_ids().len(), 1, "{status:?}");
let StartStep::Queued(write) = started else {
panic!("{status:?} queues START");
};
write.await.expect("START is written");
let updates = client.recorded_updates();
assert_eq!(updates.len(), 1, "{status:?}");
assert_eq!(updates[0].action(), &OperationAction::Start, "{status:?}");
}
let hooks = OpHooks::default();
let ctx = context(
vec![step_record(CheckpointStatus::Started, 1, "Step")],
in_memory(),
Duration::ZERO,
Some(&hooks),
);
let started = step_start(&ctx, StepSemantics::AtLeastOncePerRetry).expect("no error");
assert!(matches!(started, StartStep::Recorded(_)));
assert_eq!(ctx.queued_update_ids(), Vec::<String>::new());
}
#[derive(Default)]
struct LoggingClient(Arc<Mutex<Vec<String>>>);
impl ExecutionClient for LoggingClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let starts: Vec<String> = updates
.iter()
.filter(|u| u.action() == &OperationAction::Start)
.map(|u| u.name().unwrap_or("-").to_owned())
.collect();
let log = Arc::clone(&self.0);
Box::pin(async move {
tokio::task::yield_now().await;
for name in starts {
log.lock().unwrap().push(format!("response {name}"));
}
Ok(CheckpointOutput::new(
Some("t".to_owned()),
Vec::new(),
None,
))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
}
}
#[tokio::test]
async fn at_most_once_retry_start_hook_fires_after_start_response() {
let client = Arc::new(LoggingClient::default());
let log = Arc::clone(&client.0);
let hooks = OpHooks::probing(move || {
log.lock()
.unwrap()
.iter()
.any(|e| e == "response a")
.to_string()
});
let ctx = context(
vec![step_record(CheckpointStatus::Ready, 1, "Step")],
client,
Duration::ZERO,
Some(&hooks),
);
let op_id = ctx.mint_id();
let value = crate::step::StepExecution {
ctx: ctx.clone(),
op_id,
name: Some("a".to_owned()),
retry_strategy: None,
serdes: crate::JsonSerdes,
semantics: StepSemantics::AtMostOncePerRetry,
sub_type: crate::OperationSubType::STEP,
plugin_name: None,
closure: |_| async { Ok(1_u32) },
_marker: std::marker::PhantomData,
}
.execute()
.await
.expect("the step succeeds");
assert_eq!(value, 1);
let start = hooks
.records()
.into_iter()
.find(|r| r.hook == "start")
.expect("the start hook fired");
assert!(start.replay, "the retry's start hook is a replay");
assert_eq!(start.probe, "true", "response a came before the hook");
}
type Submission = (
String,
String,
OperationAction,
Arc<std::sync::atomic::AtomicBool>,
);
struct FailStartClient {
target: &'static str,
}
fn start_rejection() -> ExecutionClientError {
ExecutionClientError::new(
crate::client::ExecutionClientErrorScope::Execution,
"START rejected",
)
}
impl ExecutionClient for FailStartClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let fails = updates
.iter()
.any(|u| u.action() == &OperationAction::Start && u.name() == Some(self.target));
Box::pin(async move {
if fails {
Err(start_rejection())
} else {
Ok(CheckpointOutput::new(
Some("t".to_owned()),
Vec::new(),
None,
))
}
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
}
}
#[derive(Debug, Clone, Copy)]
enum Settler {
ChildSuccess,
ChildFailure,
BatchParentEmpty,
BatchParentAfterItems,
BatchItemSuccess,
BatchItemFailure,
ChildSerializeFailure,
BatchItemSerializeFailure,
}
struct FailSerialize;
impl<T: Send + 'static> crate::Serdes<T> for FailSerialize {
fn serialize(
&self,
_value: T,
_context: crate::serdes::SerdesContext,
) -> impl Future<Output = Result<String, crate::BoxError>> + Send {
std::future::ready(Err("cannot serialize".into()))
}
fn deserialize(
&self,
_wire: String,
_context: crate::serdes::SerdesContext,
) -> impl Future<Output = Result<T, crate::BoxError>> + Send {
std::future::ready(Err("cannot deserialize".into()))
}
}
impl Settler {
fn target(self) -> &'static str {
match self {
Self::ChildSuccess | Self::ChildFailure | Self::ChildSerializeFailure => "c",
Self::BatchParentEmpty | Self::BatchParentAfterItems => "m",
Self::BatchItemSuccess
| Self::BatchItemFailure
| Self::BatchItemSerializeFailure => "map-item-0",
}
}
fn action(self) -> OperationAction {
match self {
Self::ChildFailure
| Self::BatchItemFailure
| Self::ChildSerializeFailure
| Self::BatchItemSerializeFailure => OperationAction::Fail,
_ => OperationAction::Succeed,
}
}
fn has_body(self) -> bool {
!matches!(self, Self::BatchParentEmpty)
}
fn future(
self,
ctx: &DurableContext,
log: Arc<Mutex<Vec<String>>>,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
let ready = move || log.lock().unwrap().push("result ready".to_owned());
match self {
Self::ChildSuccess | Self::ChildFailure => {
let fail = matches!(self, Self::ChildFailure);
let future = ctx
.run_in_child_context(move |_child| async move {
ready();
if fail {
Err::<u32, crate::BoxError>("boom".into())
} else {
Ok(1_u32)
}
})
.name("c")
.into_future();
Box::pin(async move {
let _ignored = future.await;
})
}
Self::ChildSerializeFailure => {
let future = ctx
.run_in_child_context(move |_child| async move {
ready();
Ok(1_u32)
})
.name("c")
.serdes(FailSerialize)
.into_future();
Box::pin(async move {
let _ignored = future.await;
})
}
Self::BatchItemSerializeFailure => {
let ready = Arc::new(ready);
let future = ctx
.map(vec![1_u32], move |_child, item, _| {
let ready = Arc::clone(&ready);
async move {
ready();
Ok(item)
}
})
.name("m")
.serdes(FailSerialize)
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_percentage(100.0)
.expect("a valid percentage"),
)
.into_future();
Box::pin(async move {
let _ignored = future.await;
})
}
Self::BatchParentEmpty => {
let future = ctx
.map(Vec::<u32>::new(), |_child, item, _| async move { Ok(item) })
.name("m")
.into_future();
Box::pin(async move {
let _ignored = future.await;
})
}
Self::BatchParentAfterItems | Self::BatchItemSuccess | Self::BatchItemFailure => {
let fail = matches!(self, Self::BatchItemFailure);
let nesting = if matches!(self, Self::BatchParentAfterItems) {
crate::map_parallel::NestingMode::Flat
} else {
crate::map_parallel::NestingMode::Normal
};
let ready = Arc::new(ready);
let future = ctx
.map(vec![1_u32], move |_child, item, _| {
let ready = Arc::clone(&ready);
async move {
ready();
if fail {
Err::<u32, crate::BoxError>("boom".into())
} else {
Ok(item)
}
}
})
.name("m")
.nesting(nesting)
.completion(
crate::builders::map_parallel::CompletionConfig::with_tolerated_failure_percentage(100.0)
.expect("a valid percentage"),
)
.into_future();
Box::pin(async move {
let _ignored = future.await;
})
}
}
}
}
async fn wait_until(what: &str, mut ready: impl FnMut() -> bool) {
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
while !ready() {
assert!(tokio::time::Instant::now() < deadline, "timed out: {what}");
tokio::time::sleep(Duration::from_millis(1)).await;
}
}
#[expect(clippy::too_many_lines)] async fn outcome_waits_for_failed_start(case: Settler) {
let target = case.target();
let ctx = context(
Vec::new(),
Arc::new(FailStartClient { target }),
Duration::ZERO,
None,
);
let gate = Arc::new(tokio::sync::Notify::new());
let held = Arc::clone(&gate);
ctx.set_pre_lock_gate(Arc::new(move |updates: &[OperationUpdate]| {
updates
.iter()
.any(|u| u.action() == &OperationAction::Start && u.name() == Some(target))
.then(|| Arc::clone(&held))
}));
let submissions: Arc<Mutex<Vec<Submission>>> = Arc::default();
let recorded = Arc::clone(&submissions);
ctx.set_submission_recorder(Arc::new(move |updates, flag| {
let mut recorded = recorded.lock().unwrap();
for u in updates {
recorded.push((
u.name().unwrap_or("-").to_owned(),
u.id().to_owned(),
u.action().clone(),
Arc::clone(flag),
));
}
}));
let entries: Arc<Mutex<Vec<(String, OperationAction)>>> = Arc::default();
let entered = Arc::clone(&entries);
ctx.set_settler_entry_recorder(Arc::new(move |wire_id, action| {
entered.lock().unwrap().push((wire_id.to_owned(), action));
}));
let log: Arc<Mutex<Vec<String>>> = Arc::default();
let operation = case.future(&ctx, Arc::clone(&log));
let start_of = |submissions: &[Submission]| {
submissions
.iter()
.find(|(name, _, action, _)| name == target && action == &OperationAction::Start)
.map(|(_, id, _, flag)| (id.clone(), Arc::clone(flag)))
};
let outcomes_of = |submissions: &[Submission], id: &str| {
submissions
.iter()
.filter(|(_, sid, action, _)| {
sid == id && matches!(action, OperationAction::Succeed | OperationAction::Fail)
})
.count()
};
let observe = async {
wait_until("the target's START is submitted", || {
start_of(&submissions.lock().unwrap()).is_some()
})
.await;
let (id, flag) = start_of(&submissions.lock().unwrap()).expect("checked above");
let action = case.action();
wait_until("the settler-entry marker", || {
entries
.lock()
.unwrap()
.contains(&(id.clone(), action.clone()))
})
.await;
if case.has_body() {
wait_until("the body's result", || {
log.lock().unwrap().iter().any(|e| e == "result ready")
})
.await;
}
assert!(
flag.load(std::sync::atomic::Ordering::SeqCst),
"{case:?}: the outcome path polls the retained START"
);
assert_eq!(
outcomes_of(&submissions.lock().unwrap(), &id),
0,
"{case:?}: no outcome while START is held"
);
gate.notify_one();
wait_until("the invocation's fatal error", || {
ctx.coordinator().fatal_error().is_some()
})
.await;
let fatal = ctx.coordinator().fatal_error().expect("checked above");
let wire = crate::error::checkpoint_execution_failure_wire(&start_rejection());
assert_eq!(
fatal.error_type, "CheckpointUnrecoverableExecutionError",
"{case:?}"
);
assert_eq!(
Some(fatal.error_message.as_str()),
wire.error_message(),
"{case:?}"
);
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(
outcomes_of(&submissions.lock().unwrap(), &id),
0,
"{case:?}: no outcome after the failed START"
);
};
tokio::select! {
() = operation => panic!("{case:?}: the operation resolved after a failed START"),
() = observe => {}
}
}
#[tokio::test]
async fn child_success_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::ChildSuccess).await;
}
#[tokio::test]
async fn child_failure_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::ChildFailure).await;
}
#[tokio::test]
async fn batch_parent_success_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::BatchParentEmpty).await;
}
#[tokio::test]
async fn batch_parent_nonempty_success_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::BatchParentAfterItems).await;
}
#[tokio::test]
async fn batch_item_success_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::BatchItemSuccess).await;
}
#[tokio::test]
async fn batch_item_failure_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::BatchItemFailure).await;
}
#[tokio::test]
async fn child_serialize_failure_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::ChildSerializeFailure).await;
}
#[tokio::test]
async fn batch_item_serialize_failure_waits_for_failed_start() {
outcome_waits_for_failed_start(Settler::BatchItemSerializeFailure).await;
}
}