use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::Poll;
use std::time::Duration;
use crate::coordinator::{InvocationCoordinator, Phase};
#[cfg(test)]
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus, EngineState};
pub(crate) const UNTRACKED_DEADLINE_WARNING: &str = "the invocation reached its deadline while \
the handler awaited work the SDK does not track; run that work in a step";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum InvocationOutcome {
Complete(String),
Pending,
Failed {
error: crate::error::WireError,
},
Fault {
error_type: String,
message: String,
},
}
#[derive(Debug, Clone)]
pub(crate) struct FatalError {
pub(crate) error_type: String,
pub(crate) error_message: String,
pub(crate) severity: FatalSeverity,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FatalSeverity {
FailExecution,
FailInvocation,
}
impl FatalError {
pub(crate) fn into_outcome(self) -> InvocationOutcome {
match self.severity {
FatalSeverity::FailExecution => InvocationOutcome::Failed {
error: crate::error::WireError::new(
Some(self.error_type),
Some(self.error_message),
),
},
FatalSeverity::FailInvocation => InvocationOutcome::Fault {
error_type: self.error_type,
message: self.error_message,
},
}
}
}
#[derive(Debug)]
pub(crate) struct TaskOwnership {
owner: Owner,
blessed_tasks: std::sync::Mutex<std::collections::HashSet<tokio::task::Id>>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum Owner {
Task(tokio::task::Id),
Thread(std::thread::ThreadId),
}
impl TaskOwnership {
pub(crate) fn new_current() -> Self {
let owner = match tokio::task::try_id() {
Some(id) => Owner::Task(id),
None => Owner::Thread(std::thread::current().id()),
};
Self {
owner,
blessed_tasks: std::sync::Mutex::new(std::collections::HashSet::new()),
}
}
pub(crate) fn bless_task(&self, task_id: tokio::task::Id) {
let mut blessed = self
.blessed_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
blessed.insert(task_id);
}
pub(crate) fn unbless_task(&self, task_id: tokio::task::Id) {
self.blessed_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&task_id);
}
#[cfg(test)]
pub(crate) fn blessed_count(&self) -> usize {
self.blessed_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len()
}
pub(crate) fn check_current_task(&self) -> Result<(), String> {
if let Some(current_id) = tokio::task::try_id() {
if matches!(self.owner, Owner::Task(owner_id) if owner_id == current_id) {
return Ok(());
}
let blessed = self
.blessed_tasks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if blessed.contains(¤t_id) {
return Ok(());
}
Err(format!(
"durable operation invoked from task {current_id:?}, which does not own the \
context (owner: {:?}). Use .spawn() instead of tokio::spawn for durable \
fan-out",
self.owner
))
} else {
let current = std::thread::current().id();
if matches!(self.owner, Owner::Thread(owner) if owner == current) {
return Ok(());
}
Err(format!(
"durable operation invoked from thread {current:?}, which does not own the \
context (owner: {:?}). Use .spawn() instead of std::thread::spawn for durable \
fan-out",
self.owner
))
}
}
}
enum DriveEnd<T> {
Completed(T),
Interrupted,
}
struct PanicOwnershipGuard(Arc<InvocationCoordinator>);
impl Drop for PanicOwnershipGuard {
fn drop(&mut self) {
if let Some(payload) = self.0.take_panic() {
crate::task::discard_panic(payload);
}
}
}
struct TeardownGuard(Arc<InvocationCoordinator>);
impl Drop for TeardownGuard {
fn drop(&mut self) {
self.0.finish();
self.0.tasks().close();
}
}
pub(crate) async fn drive_invocation<F>(
handler_future: F,
root_cx: crate::task::TaskContext,
task_ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
root: crate::DurableContext,
remaining: Option<Duration>,
) -> Result<(InvocationOutcome, bool), Box<dyn std::any::Any + Send + 'static>>
where
F: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
{
drive_with_scheduler(
root_cx,
handler_future,
task_ownership,
coordinator,
refresh_scheduler(root),
remaining,
)
.await
}
async fn drive_with_scheduler<F, S>(
root_cx: crate::task::TaskContext,
handler_future: F,
task_ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
scheduler: S,
remaining: Option<Duration>,
) -> Result<(InvocationOutcome, bool), Box<dyn std::any::Any + Send + 'static>>
where
F: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
S: Future<Output = ()>,
{
let ownership_guard = PanicOwnershipGuard(Arc::clone(&coordinator));
let _teardown = TeardownGuard(Arc::clone(&coordinator));
coordinator.start_deadline(remaining);
let mut scheduler = Box::pin(scheduler);
if coordinator.has_panic() {
coordinator.finish();
}
#[expect(clippy::expect_used)]
let (mut root, root_join) =
crate::task::spawn_root(root_cx, &task_ownership, &coordinator, handler_future)
.expect("the invocation driver runs inside a Tokio runtime");
let mut deadline = remaining.map(|d| Box::pin(tokio::time::sleep(d)));
let ended = std::future::poll_fn(|cx| {
coordinator.register_driver_waker(cx.waker());
if coordinator.fatal_error().is_some() || coordinator.has_panic() {
return Poll::Ready(DriveEnd::Interrupted);
}
match Pin::new(&mut root).poll(cx) {
Poll::Ready(crate::task::TaskOutcome::Value(result)) => {
return Poll::Ready(DriveEnd::Completed(result));
}
Poll::Ready(crate::task::TaskOutcome::Panicked(payload)) => {
crate::task::record_or_discard(&coordinator, payload);
return Poll::Ready(DriveEnd::Interrupted);
}
Poll::Pending => {}
}
if coordinator.phase() == Phase::Suspending {
return Poll::Ready(DriveEnd::Interrupted);
}
if let Err(payload) =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| scheduler.as_mut().poll(cx)))
{
crate::task::record_or_discard(&coordinator, payload);
return Poll::Ready(DriveEnd::Interrupted);
}
if let Some(sleep) = deadline.as_mut()
&& sleep.as_mut().poll(cx).is_ready()
{
deadline = None;
if coordinator.running_and_idle() {
tracing::warn!("{UNTRACKED_DEADLINE_WARNING}");
}
}
Poll::Pending
})
.await;
crate::task::drop_recorded(&coordinator, scheduler);
coordinator.finish();
root.abort();
drop(root);
drop(root_join.await);
let tasks = Arc::clone(coordinator.tasks());
tasks.abort_all();
tasks.wait_idle().await;
tasks.claim_unread_panics(&coordinator);
let fatal = coordinator.fatal_error();
let panic = coordinator.take_panic();
drop(ownership_guard);
#[cfg(test)]
test_hooks::after_arbitration(&coordinator);
if let Some(fatal) = fatal {
if let Some(payload) = panic {
crate::task::discard_panic(payload);
}
return Ok((fatal.into_outcome(), true));
}
if let Some(payload) = panic {
return Err(payload);
}
let outcome = match ended {
DriveEnd::Completed(Ok(result)) => InvocationOutcome::Complete(result),
DriveEnd::Completed(Err(err)) => InvocationOutcome::Failed { error: err },
DriveEnd::Interrupted => InvocationOutcome::Pending,
};
Ok((outcome, false))
}
pub(crate) fn withdrawal_answer(
ctx: &crate::DurableContext,
response: Result<crate::InvocationResponse, lambda_runtime::Error>,
end: crate::InvocationEnd,
) -> (
Result<crate::InvocationResponse, lambda_runtime::Error>,
crate::InvocationEnd,
) {
if !ctx.coordinator().withdrawn() {
return (response, end);
}
if end.status == crate::InvocationStatus::Pending {
return (response, end);
}
drop(response);
let (pending, mut pending_end) = crate::outcome_envelope(InvocationOutcome::Pending, ctx);
pending_end.error = end.error;
(pending, pending_end)
}
#[cfg(test)]
pub(crate) async fn drive_outcome<F>(
handler_future: F,
task_ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
root: crate::DurableContext,
remaining: Option<Duration>,
) -> Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>
where
F: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
{
drive_invocation(
handler_future,
crate::task::TaskContext::current(),
task_ownership,
coordinator,
root,
remaining,
)
.await
.map(|(outcome, _)| outcome)
}
#[cfg(test)]
pub(crate) mod test_hooks {
tokio::task_local! {
pub(crate) static AFTER_ARBITRATION: std::sync::Arc<
dyn Fn(&super::InvocationCoordinator) + Send + Sync,
>;
}
pub(crate) fn after_arbitration(coordinator: &super::InvocationCoordinator) {
if let Ok(hook) = AFTER_ARBITRATION.try_with(std::sync::Arc::clone) {
hook(coordinator);
}
}
}
async fn refresh_scheduler(root: crate::DurableContext) {
use crate::coordinator::{DueRefresh, PermitKind};
let coordinator = Arc::clone(root.coordinator());
loop {
match coordinator.next_refresh_at() {
None => {
coordinator.rearmed().await;
continue;
}
Some(at) => {
tokio::select! {
biased;
() = coordinator.rearmed() => continue,
() = tokio::time::sleep_until(at) => {}
}
}
}
let (keys, first_wire_id) = match coordinator.take_due(tokio::time::Instant::now()) {
DueRefresh::Nothing | DueRefresh::UnderFloor => continue,
DueRefresh::Due {
keys,
first_wire_id,
} => (keys, first_wire_id),
};
let Some(permit) = coordinator.acquire(PermitKind::Scheduler) else {
coordinator.rearmed().await;
continue;
};
root.checkpoint_refresh(&first_wire_id).await;
let records = keys
.iter()
.filter_map(|(wire_id, _)| {
root.refresh_view(wire_id)
.map(|view| (wire_id.clone(), view))
})
.collect();
coordinator.after_refresh(&keys, &records);
drop(permit);
}
}
#[cfg(test)]
pub(crate) mod test_support {
use super::{InvocationOutcome, TaskOwnership, drive_outcome};
use crate::client::{CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput};
use crate::coordinator::InvocationCoordinator;
use aws_sdk_lambda::types::{Operation, OperationStatus, OperationUpdate};
use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::time::Duration;
pub(crate) async fn drive_ok<F>(
root: crate::DurableContext,
fut: F,
ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
) -> InvocationOutcome
where
F: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
{
match drive_outcome(fut, ownership, coordinator, root, None).await {
Ok(outcome) => outcome,
Err(payload) => std::panic::resume_unwind(payload),
}
}
pub(crate) async fn outcome_of<F>(
root: crate::DurableContext,
ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
fut: F,
) -> InvocationOutcome
where
F: IntoFuture + Send + 'static,
F::IntoFuture: Send + 'static,
{
let fut = Box::pin(async move {
let _ = fut.await;
Ok::<_, crate::error::WireError>("ok".to_owned())
});
match drive_outcome(fut, ownership, coordinator, root, None).await {
Ok(outcome) => outcome,
Err(payload) => std::panic::resume_unwind(payload),
}
}
#[expect(clippy::panic)] pub(crate) async fn expect_serdes_fault<F>(
root: crate::DurableContext,
ownership: Arc<TaskOwnership>,
coordinator: Arc<InvocationCoordinator>,
fut: F,
needle: &str,
) where
F: IntoFuture + Send + 'static,
F::IntoFuture: Send + 'static,
{
let outcome = tokio::time::timeout(
Duration::from_secs(5),
outcome_of(root, ownership, coordinator, fut),
)
.await
.expect("the operation must reach an invocation outcome within 5 s");
let InvocationOutcome::Fault { message, .. } = outcome else {
panic!("expected an invocation fault, got {outcome:?}");
};
assert!(
message.contains(needle),
"fault message {message:?} must contain {needle:?}"
);
}
type Responder = dyn Fn(&[OperationUpdate], &[OperationUpdate]) -> Vec<Operation> + Send + Sync;
pub(crate) struct ScriptedClient {
respond: Box<Responder>,
delay: Option<Duration>,
first_call_gate: Option<Arc<tokio::sync::Notify>>,
calls: Mutex<Vec<(String, Vec<OperationUpdate>)>>,
tokens: Mutex<u32>,
}
impl ScriptedClient {
pub(crate) fn new(
respond: impl Fn(&[OperationUpdate], &[OperationUpdate]) -> Vec<Operation>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
respond: Box::new(respond),
delay: None,
first_call_gate: None,
calls: Mutex::new(Vec::new()),
tokens: Mutex::new(0),
}
}
pub(crate) fn empty() -> Self {
Self::new(|_, _| Vec::new())
}
pub(crate) fn with_delay(mut self, delay: Duration) -> Self {
self.delay = Some(delay);
self
}
pub(crate) fn with_first_call_gate(mut self, gate: Arc<tokio::sync::Notify>) -> Self {
self.first_call_gate = Some(gate);
self
}
pub(crate) fn calls(&self) -> Vec<(String, Vec<OperationUpdate>)> {
self.calls.lock().expect("lock").clone()
}
pub(crate) fn updates(&self) -> Vec<OperationUpdate> {
self.calls().into_iter().flat_map(|(_, u)| u).collect()
}
pub(crate) fn call_count(&self) -> usize {
self.calls.lock().expect("lock").len()
}
pub(crate) fn received(
&self,
name: &str,
action: &aws_sdk_lambda::types::OperationAction,
) -> bool {
self.updates()
.iter()
.any(|u| u.name() == Some(name) && u.action() == action)
}
}
impl ExecutionClient for ScriptedClient {
fn checkpoint(
&self,
_execution_arn: &str,
checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let (operations, first) = {
let mut calls = self.calls.lock().expect("lock");
let mut history: Vec<OperationUpdate> =
calls.iter().flat_map(|(_, u)| u.iter().cloned()).collect();
history.extend(updates.iter().cloned());
let operations = (self.respond)(&updates, &history);
calls.push((checkpoint_token.to_owned(), updates));
(operations, calls.len() == 1)
};
let token = {
let mut tokens = self.tokens.lock().expect("lock");
*tokens += 1;
format!("t{tokens}")
};
let delay = self.delay;
let gate = if first {
self.first_call_gate.clone()
} else {
None
};
Box::pin(async move {
if let Some(gate) = gate {
gate.notified().await;
}
if let Some(delay) = delay {
tokio::time::sleep(delay).await;
}
Ok(CheckpointOutput::new(Some(token), operations, None))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
}
}
pub(crate) async fn poll_once<F: Future>(mut f: Pin<&mut F>) {
std::future::poll_fn(|cx| {
let _ = f.as_mut().poll(cx);
std::task::Poll::Ready(())
})
.await;
}
pub(crate) struct RefreshRequest<'a> {
pub(crate) updates: &'a [OperationUpdate],
pub(crate) history: &'a [OperationUpdate],
pub(crate) empty_number: usize,
pub(crate) now: Duration,
pub(crate) base: std::time::SystemTime,
}
impl RefreshRequest<'_> {
pub(crate) fn timestamp_in(&self, in_: Duration) -> aws_smithy_types::DateTime {
aws_smithy_types::DateTime::from(self.base + self.now + in_)
}
}
type RefreshResponder =
dyn Fn(&RefreshRequest<'_>) -> Result<Vec<Operation>, ExecutionClientError> + Send + Sync;
pub(crate) struct RefreshClient {
respond: Box<RefreshResponder>,
empty_delay: Duration,
start: tokio::time::Instant,
base: std::time::SystemTime,
calls: Mutex<Vec<(Duration, Vec<OperationUpdate>)>>,
empty_answers: Mutex<Vec<Duration>>,
tokens: Mutex<u32>,
}
impl RefreshClient {
pub(crate) fn new(
respond: impl Fn(&RefreshRequest<'_>) -> Result<Vec<Operation>, ExecutionClientError>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
respond: Box::new(respond),
empty_delay: Duration::ZERO,
start: tokio::time::Instant::now(),
base: std::time::SystemTime::now(),
calls: Mutex::new(Vec::new()),
empty_answers: Mutex::new(Vec::new()),
tokens: Mutex::new(0),
}
}
pub(crate) fn with_empty_delay(mut self, delay: Duration) -> Self {
self.empty_delay = delay;
self
}
pub(crate) fn elapsed(&self) -> Duration {
self.start.elapsed()
}
pub(crate) fn empty_calls(&self) -> Vec<Duration> {
self.calls
.lock()
.expect("lock")
.iter()
.filter(|(_, updates)| updates.is_empty())
.map(|(at, _)| *at)
.collect()
}
pub(crate) fn empty_answers(&self) -> Vec<Duration> {
self.empty_answers.lock().expect("lock").clone()
}
pub(crate) fn time_of(
&self,
name: &str,
action: &aws_sdk_lambda::types::OperationAction,
) -> Option<Duration> {
self.calls
.lock()
.expect("lock")
.iter()
.find(|(_, updates)| named(updates, name, action).is_some())
.map(|(at, _)| *at)
}
pub(crate) fn updates(&self) -> Vec<OperationUpdate> {
self.calls
.lock()
.expect("lock")
.iter()
.flat_map(|(_, u)| u.iter().cloned())
.collect()
}
pub(crate) fn received(
&self,
name: &str,
action: &aws_sdk_lambda::types::OperationAction,
) -> bool {
self.time_of(name, action).is_some()
}
}
impl ExecutionClient for RefreshClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let (history, empty_number) = {
let mut calls = self.calls.lock().expect("lock");
calls.push((self.start.elapsed(), updates.clone()));
let history: Vec<OperationUpdate> =
calls.iter().flat_map(|(_, u)| u.iter().cloned()).collect();
let empty_number = if updates.is_empty() {
calls.iter().filter(|(_, u)| u.is_empty()).count()
} else {
0
};
(history, empty_number)
};
let token = {
let mut tokens = self.tokens.lock().expect("lock");
*tokens += 1;
format!("t{tokens}")
};
Box::pin(async move {
if empty_number > 0 && !self.empty_delay.is_zero() {
tokio::time::sleep(self.empty_delay).await;
}
let now = self.start.elapsed();
let operations = (self.respond)(&RefreshRequest {
updates: &updates,
history: &history,
empty_number,
now,
base: self.base,
})?;
if empty_number > 0 {
self.empty_answers.lock().expect("lock").push(now);
}
Ok(CheckpointOutput::new(Some(token), operations, None))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
}
}
pub(crate) fn refresh_ctx(client: &Arc<RefreshClient>) -> crate::DurableContext {
crate::DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(client) as Arc<dyn ExecutionClient>,
"t0".to_owned(),
)
}
pub(crate) fn callback_record(
start: &OperationUpdate,
status: OperationStatus,
result: Option<&str>,
) -> Operation {
let mut details = aws_sdk_lambda::types::CallbackDetails::builder().callback_id("cb-1");
if let Some(result) = result {
details = details.result(result);
}
operation_like(start, status)
.callback_details(details.build())
.build()
.expect("all required fields set")
}
pub(crate) fn callback_starts(updates: &[OperationUpdate]) -> Vec<Operation> {
updates
.iter()
.filter(|u| {
u.r#type() == &aws_sdk_lambda::types::OperationType::Callback
&& u.action() == &aws_sdk_lambda::types::OperationAction::Start
})
.map(|start| callback_record(start, OperationStatus::Started, None))
.collect()
}
#[track_caller]
pub(crate) fn assert_near(actual: Duration, expected: Duration) {
let diff = actual.abs_diff(expected);
assert!(
diff <= Duration::from_millis(50),
"expected {expected:?}, got {actual:?}"
);
}
#[track_caller]
pub(crate) fn assert_times(actual: &[Duration], expected: &[Duration]) {
assert_eq!(
actual.len(),
expected.len(),
"expected times {expected:?}, got {actual:?}"
);
for (a, e) in actual.iter().zip(expected) {
assert_near(*a, *e);
}
}
pub(crate) fn scripted_ctx(client: &Arc<ScriptedClient>) -> crate::DurableContext {
crate::DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(client) as Arc<dyn ExecutionClient>,
"t0".to_owned(),
)
}
pub(crate) fn operation_like(
update: &OperationUpdate,
status: OperationStatus,
) -> aws_sdk_lambda::types::builders::OperationBuilder {
let mut builder = Operation::builder()
.id(update.id())
.r#type(update.r#type().clone())
.status(status)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0));
if let Some(sub_type) = update.sub_type() {
builder = builder.sub_type(sub_type);
}
if let Some(name) = update.name() {
builder = builder.name(name);
}
if let Some(parent) = update.parent_id() {
builder = builder.parent_id(parent);
}
builder
}
pub(crate) fn named<'a>(
updates: &'a [OperationUpdate],
name: &str,
action: &aws_sdk_lambda::types::OperationAction,
) -> Option<&'a OperationUpdate> {
updates
.iter()
.find(|u| u.name() == Some(name) && u.action() == action)
}
pub(crate) async fn drive_handler<F, Fut>(
ctx: crate::DurableContext,
remaining: Option<Duration>,
handler: F,
) -> Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>
where
F: FnOnce(crate::DurableContext) -> Fut,
Fut: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
{
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let handler = Box::pin(handler(ctx));
drive_outcome(handler, ownership, coordinator, root, remaining).await
}
#[derive(Clone, Default)]
pub(crate) struct EventMessages(Arc<Mutex<Vec<(tracing::Level, String)>>>);
struct MessageVisitor(String);
impl tracing::field::Visit for MessageVisitor {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
if field.name() == "message" {
self.0 = format!("{value:?}");
}
}
}
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for EventMessages {
fn on_event(
&self,
event: &tracing::Event<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
) {
let mut visitor = MessageVisitor(String::new());
event.record(&mut visitor);
self.0
.lock()
.expect("lock")
.push((*event.metadata().level(), visitor.0));
}
}
impl EventMessages {
pub(crate) fn install() -> (Self, tracing::subscriber::DefaultGuard) {
use tracing_subscriber::layer::SubscriberExt as _;
let capture = Self::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
(capture, test_tracing::install_test_subscriber(subscriber))
}
pub(crate) fn at(&self, level: tracing::Level) -> Vec<String> {
self.0
.lock()
.expect("lock")
.iter()
.filter(|(l, _)| *l == level)
.map(|(_, m)| m.clone())
.collect()
}
pub(crate) fn replayed(&self) -> usize {
self.at(tracing::Level::DEBUG)
.iter()
.filter(|m| *m == crate::observability::event_names::OPERATION_REPLAYED)
.count()
}
}
#[expect(clippy::needless_pass_by_value)]
pub(crate) fn wire_err(err: crate::OperationError) -> crate::error::WireError {
crate::error::WireError::new(None::<String>, Some(format!("{err:#}")))
}
#[derive(Clone)]
pub(crate) struct LatencyRequest {
pub(crate) received: Duration,
pub(crate) answered: Option<Duration>,
pub(crate) updates: Vec<OperationUpdate>,
}
pub(crate) struct LatencyClient {
latency: Duration,
fail_start: bool,
start: tokio::time::Instant,
requests: Arc<Mutex<Vec<LatencyRequest>>>,
tokens: Mutex<u32>,
}
impl LatencyClient {
pub(crate) fn new(latency: Duration) -> Self {
Self {
latency,
fail_start: false,
start: tokio::time::Instant::now(),
requests: Arc::new(Mutex::new(Vec::new())),
tokens: Mutex::new(0),
}
}
pub(crate) fn failing_start(mut self) -> Self {
self.fail_start = true;
self
}
pub(crate) fn elapsed(&self) -> Duration {
self.start.elapsed()
}
pub(crate) fn requests(&self) -> Vec<LatencyRequest> {
self.requests.lock().expect("lock").clone()
}
pub(crate) fn request_with(
&self,
action: &aws_sdk_lambda::types::OperationAction,
) -> Option<LatencyRequest> {
self.requests()
.into_iter()
.find(|r| r.updates.iter().any(|u| u.action() == action))
}
pub(crate) fn received_outcome(&self) -> bool {
use aws_sdk_lambda::types::OperationAction;
self.requests()
.iter()
.flat_map(|r| r.updates.clone())
.any(|u| {
matches!(
u.action(),
OperationAction::Succeed | OperationAction::Retry | OperationAction::Fail
)
})
}
}
impl ExecutionClient for LatencyClient {
fn checkpoint(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
use aws_sdk_lambda::types::OperationAction;
let index = {
let mut requests = self.requests.lock().expect("lock");
requests.push(LatencyRequest {
received: self.start.elapsed(),
answered: None,
updates: updates.clone(),
});
requests.len() - 1
};
let token = {
let mut tokens = self.tokens.lock().expect("lock");
*tokens += 1;
format!("t{tokens}")
};
Box::pin(async move {
tokio::time::sleep(self.latency).await;
if let Some(request) = self.requests.lock().expect("lock").get_mut(index) {
request.answered = Some(self.start.elapsed());
}
let starts: Vec<&OperationUpdate> = updates
.iter()
.filter(|u| u.action() == &OperationAction::Start)
.collect();
if self.fail_start && !starts.is_empty() {
return Err(ExecutionClientError::new_non_retryable("START rejected"));
}
let operations = starts
.into_iter()
.map(|u| {
operation_like(u, OperationStatus::Started)
.build()
.expect("all required fields set")
})
.collect();
Ok(CheckpointOutput::new(Some(token), operations, None))
})
}
fn get_state(
&self,
_execution_arn: &str,
_checkpoint_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async { Ok(GetStateOutput::new(Vec::new(), None)) })
}
}
pub(crate) fn latency_ctx(
client: &Arc<LatencyClient>,
buffer_window: Duration,
) -> crate::DurableContext {
crate::DurableContext::new_root_with_client_and_defaults(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(client) as Arc<dyn ExecutionClient>,
"t0".to_owned(),
buffer_window,
0,
None,
)
}
}
#[cfg(test)]
#[expect(clippy::disallowed_methods)]
mod tests {
use super::test_support::drive_ok;
use super::*;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
struct DropGuard {
dropped: Arc<AtomicBool>,
}
impl Drop for DropGuard {
fn drop(&mut self) {
self.dropped.store(true, Ordering::SeqCst);
}
}
#[test]
fn capture_sees_callsite_first_reached_without_thread_default() {
fn probe() {
tracing::info!("capture-probe");
}
let (capture, _guard) = test_support::EventMessages::install();
std::thread::spawn(probe)
.join()
.expect("the probe thread does not panic");
assert_eq!(capture.at(tracing::Level::INFO), Vec::<String>::new());
probe();
assert_eq!(capture.at(tracing::Level::INFO), ["capture-probe"]);
}
fn waiting_ctx() -> crate::DurableContext {
let client = Arc::new(test_support::ScriptedClient::empty());
test_support::scripted_ctx(&client)
}
#[tokio::test]
async fn suspension_drops_future_at_await_point() {
let ctx = waiting_ctx();
let drop_observed = Arc::new(AtomicBool::new(false));
let after_await_ran = Arc::new(AtomicBool::new(false));
let drop_clone = Arc::clone(&drop_observed);
let after_clone = Arc::clone(&after_await_ran);
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_ok(
ctx.clone(),
async move {
let _guard = DropGuard {
dropped: drop_clone,
};
let _ = ctx.wait(Duration::from_secs(30)).await;
after_clone.store(true, Ordering::SeqCst);
Ok("should not reach".to_owned())
},
ownership,
coordinator,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
drop_observed.load(Ordering::SeqCst),
"future was not dropped at the await point"
);
assert!(
!after_await_ran.load(Ordering::SeqCst),
"code after the suspension point must not run"
);
}
#[tokio::test]
async fn suspension_unswallowable_by_catch_all() {
let ctx = waiting_ctx();
let after_catch_ran = Arc::new(AtomicBool::new(false));
let after_clone = Arc::clone(&after_catch_ran);
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_ok(
ctx.clone(),
async move {
let result: Result<String, crate::error::WireError> = async {
let _ = ctx.wait(Duration::from_secs(30)).await;
Ok("survived".to_owned())
}
.await;
after_clone.store(true, Ordering::SeqCst);
result
},
ownership,
coordinator,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!after_catch_ran.load(Ordering::SeqCst),
"catch-all must not intercept suspension — future is dropped"
);
}
#[tokio::test]
async fn pending_outcome_surfaces_to_caller() {
let ctx = waiting_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_ok(
ctx.clone(),
async move {
let _ = ctx.wait(Duration::from_secs(30)).await;
Ok("unreachable".to_owned())
},
ownership,
coordinator,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
}
#[tokio::test]
async fn driver_returns_complete_on_success() {
let ctx = crate::DurableContext::test_context();
let signal = Arc::clone(ctx.coordinator());
let outcome = drive_ok(
ctx.clone(),
async move { Ok::<_, crate::error::WireError>("done".to_owned()) },
Arc::clone(ctx.task_ownership()),
Arc::clone(&signal),
)
.await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
}
#[tokio::test]
async fn driver_returns_failed_on_error() {
let ctx = crate::DurableContext::test_context();
let signal = Arc::clone(ctx.coordinator());
let outcome = drive_ok(
ctx.clone(),
async move {
Err(crate::error::WireError::new(
Some("Error".to_owned()),
Some("boom".to_owned()),
))
},
Arc::clone(ctx.task_ownership()),
Arc::clone(&signal),
)
.await;
let InvocationOutcome::Failed { error } = outcome else {
unreachable!("expected Failed, got {outcome:?}");
};
assert_eq!(error.error_type(), Some("Error"));
assert_eq!(error.error_message(), Some("boom"));
}
#[tokio::test]
async fn resume_replays_frozen_results_without_re_executing() {
let execution_count = Arc::new(AtomicU32::new(0));
let exec_clone = Arc::clone(&execution_count);
let wire_key = crate::engine::compute_wire_id_public("1");
let log = Arc::new(CheckpointLog::from_records(vec![(
wire_key.clone(),
CheckpointRecord {
id: wire_key,
status: CheckpointStatus::Succeeded,
result: Some(r#""frozen_value""#.to_owned()),
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
},
)]));
let engine = EngineState::new_root(Arc::clone(&log));
let op_id = engine.mint_id();
let is_replay = engine.is_replaying_at(op_id.positional());
let result = if is_replay {
let wire = crate::engine::compute_wire_id_public(op_id.positional());
let record = log.get(&wire);
record.unwrap().result.clone()
} else {
exec_clone.fetch_add(1, Ordering::SeqCst);
Some(r#""live_value""#.to_owned())
};
assert_eq!(result, Some(r#""frozen_value""#.to_owned()));
assert_eq!(
execution_count.load(Ordering::SeqCst),
0,
"step body must NOT execute during replay"
);
}
#[tokio::test]
async fn ownership_check_rejects_foreign_task() {
let result = tokio::spawn(async {
let ownership = Arc::new(TaskOwnership::new_current());
let ownership_clone = Arc::clone(&ownership);
let handle = tokio::spawn(async move { ownership_clone.check_current_task() });
handle.await.unwrap()
})
.await;
let inner_result = result.unwrap();
assert!(inner_result.is_err());
let msg = inner_result.unwrap_err();
assert!(
msg.contains("Use .spawn()"),
"error message should guide users: {msg}"
);
}
#[tokio::test]
async fn ownership_check_allows_owning_task() {
let result = tokio::spawn(async {
let ownership = TaskOwnership::new_current();
ownership.check_current_task()
})
.await;
let inner = result.unwrap();
assert!(inner.is_ok(), "owning task should pass: {inner:?}");
}
#[tokio::test]
async fn ownership_spawn_exemption_blesses_task() {
let result = tokio::spawn(async {
let ownership = Arc::new(TaskOwnership::new_current());
let ownership_clone = Arc::clone(&ownership);
let handle = tokio::spawn(async move {
let task_id = tokio::task::id();
ownership_clone.bless_task(task_id);
ownership_clone.check_current_task()
});
handle.await.unwrap()
})
.await;
let inner = result.unwrap();
assert!(
inner.is_ok(),
"blessed task should pass ownership check: {inner:?}"
);
}
#[tokio::test]
async fn ownership_root_context_allows_inline_operations() {
let ownership = TaskOwnership::new_current();
assert!(
matches!(ownership.owner, Owner::Thread(_)),
"sanity: #[tokio::test] root runs in block_on with no task ID"
);
let result = ownership.check_current_task();
assert!(
result.is_ok(),
"inline operation on the owner thread should pass: {result:?}"
);
let ownership = Arc::new(ownership);
let other = Arc::clone(&ownership);
let result = std::thread::spawn(move || other.check_current_task())
.join()
.unwrap();
assert!(
result.is_err_and(|msg| msg.contains("does not own the context")),
"a thread that does not own the context must be refused"
);
}
#[test]
fn ownership_task_context_refuses_callers_without_a_task() {
let rt = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let ownership = rt.block_on(async {
tokio::spawn(async { Arc::new(TaskOwnership::new_current()) })
.await
.unwrap()
});
assert!(matches!(ownership.owner, Owner::Task(_)));
assert!(ownership.check_current_task().is_err());
}
#[tokio::test]
async fn ownership_root_context_rejects_unblessed_spawn() {
let ownership = Arc::new(TaskOwnership::new_current());
assert!(
matches!(ownership.owner, Owner::Thread(_)),
"sanity: root runs in block_on with no task ID"
);
let ownership_clone = Arc::clone(&ownership);
let handle = tokio::spawn(async move { ownership_clone.check_current_task() });
let inner = handle.await.unwrap();
assert!(
inner.is_err(),
"unblessed spawned task must be rejected in root-context mode"
);
let msg = inner.unwrap_err();
assert!(
msg.contains("Use .spawn()"),
"error message should guide users: {msg}"
);
}
#[tokio::test]
async fn ownership_root_context_allows_blessed_spawn() {
let ownership = Arc::new(TaskOwnership::new_current());
assert!(
matches!(ownership.owner, Owner::Thread(_)),
"sanity: root runs in block_on with no task ID"
);
let ownership_clone = Arc::clone(&ownership);
let handle = tokio::spawn(async move {
let task_id = tokio::task::id();
ownership_clone.bless_task(task_id);
ownership_clone.check_current_task()
});
let inner = handle.await.unwrap();
assert!(
inner.is_ok(),
"blessed task in root-context mode should pass: {inner:?}"
);
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod suspension_containment_and_task_ownership {
use super::InvocationOutcome;
use super::test_support::drive_ok;
use crate::client::{ExecutionClient, InMemoryExecutionClient, TestResponse};
use crate::context::DurableContext;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
use aws_sdk_lambda::types::OperationAction;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
fn live_ctx(client: Arc<InMemoryExecutionClient>) -> DurableContext {
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client as Arc<dyn ExecutionClient>,
"token0".to_owned(),
)
}
fn succeeded_record(wire: &str) -> CheckpointRecord {
CheckpointRecord {
id: wire.to_owned(),
status: CheckpointStatus::Succeeded,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
}
}
#[tokio::test]
async fn wait_suspend_does_not_run_following_code() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let _ = ctx_h.wait(Duration::from_secs(30)).await;
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!ran.load(Ordering::SeqCst),
"code after a suspended wait ran"
);
}
#[tokio::test]
async fn invoke_suspend_does_not_run_following_code() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let _ = ctx_h
.invoke::<serde_json::Value>("target-fn", "input".to_owned())
.await;
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!ran.load(Ordering::SeqCst),
"code after a suspended invoke ran"
);
}
#[tokio::test]
async fn step_retry_suspend_does_not_run_following_code() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let _ = ctx_h
.step(|_| async { Err::<i32, crate::BoxError>("boom".into()) })
.name("retrying")
.await;
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!ran.load(Ordering::SeqCst),
"code after a suspended step retry ran"
);
}
#[tokio::test]
async fn wait_for_condition_suspend_does_not_run_following_code() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let _ = ctx_h
.wait_for_condition(|_ctx, state: i32| async move { Ok(state + 1) }, 0)
.wait_strategy_fn(|_state: i32, _attempt| {
crate::wait_for_condition::WaitDecision::continue_with(Duration::from_secs(
1,
))
})
.await;
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!ran.load(Ordering::SeqCst),
"code after a suspended wfc ran"
);
}
#[tokio::test]
async fn callback_suspend_does_not_run_following_code() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let Ok(cb) = ctx_h.create_callback::<serde_json::Value>().await else {
return Err(crate::error::WireError::new(
Some("E".to_owned()),
Some("create failed".to_owned()),
));
};
let _ = cb.result().await;
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!ran.load(Ordering::SeqCst),
"code after a suspended callback ran"
);
}
#[tokio::test]
async fn replayed_completed_wait_returns_ok_and_continues() {
let wire = crate::engine::compute_wire_id_public("1");
let log = Arc::new(CheckpointLog::from_records(vec![(
wire.clone(),
succeeded_record(&wire),
)]));
let ctx = DurableContext::new_root(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
);
let signal = Arc::clone(ctx.coordinator());
let ran = Arc::new(AtomicBool::new(false));
let ran_c = Arc::clone(&ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
ctx_h
.wait(Duration::from_secs(30))
.await
.expect("replayed completed wait returns Ok");
ran_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(
ran.load(Ordering::SeqCst),
"code after a replayed wait must run"
);
}
#[tokio::test]
async fn non_retryable_checkpoint_failure_uncatchable_fails_execution() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(TestResponse::NonRetryableError("kaboom".to_owned()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let caught = Arc::new(AtomicBool::new(false));
let caught_c = Arc::clone(&caught);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let r = ctx_h.wait(Duration::from_secs(5)).await;
if r.is_err() {
caught_c.store(true, Ordering::SeqCst);
}
Ok::<_, crate::error::WireError>("handled".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert!(
!caught.load(Ordering::SeqCst),
"a checkpoint API failure must not be observable by a catch"
);
match outcome {
InvocationOutcome::Failed { error } => {
assert_eq!(
error.error_type(),
Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE),
"a non-retryable checkpoint failure fails the execution \
with the unrecoverable-execution type"
);
assert!(
error.error_message().is_some_and(|m| m.contains("kaboom")),
"the service's rejection reason is preserved: {error:?}"
);
}
other => panic!("expected a FAILED execution outcome, got {other:?}"),
}
}
#[tokio::test]
async fn retryable_checkpoint_failure_uncatchable_fails_invocation() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
client.enqueue_checkpoint_response(TestResponse::RetryableError("throttled".to_owned()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let caught = Arc::new(AtomicBool::new(false));
let caught_c = Arc::clone(&caught);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let r = ctx_h.wait(Duration::from_secs(5)).await;
if r.is_err() {
caught_c.store(true, Ordering::SeqCst);
}
Ok::<_, crate::error::WireError>("handled".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert!(
!caught.load(Ordering::SeqCst),
"a checkpoint API failure must not be observable by a catch"
);
match outcome {
InvocationOutcome::Fault { message, .. } => {
assert!(
message.contains("throttled"),
"the exhausted failure's reason is preserved: {message}"
);
}
other => panic!("expected an invocation Fault outcome, got {other:?}"),
}
}
#[tokio::test]
async fn dropping_spawn_handle_keeps_task_running() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let started = Arc::new(AtomicBool::new(false));
let reached_end = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
{
let s = Arc::clone(&started);
let e = Arc::clone(&reached_end);
let d = Arc::clone(&dropped);
let handle = ctx
.step(move |_| {
let s = Arc::clone(&s);
let e = Arc::clone(&e);
let d = Arc::clone(&d);
async move {
struct G(Arc<AtomicBool>);
impl Drop for G {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
let _g = G(d);
s.store(true, Ordering::SeqCst);
std::future::pending::<()>().await;
e.store(true, Ordering::SeqCst);
Ok(1i32)
}
})
.name("bg")
.spawn();
for _ in 0..3 {
tokio::task::yield_now().await;
}
assert!(
started.load(Ordering::SeqCst),
"spawned task should have started"
);
drop(handle);
}
for _ in 0..5 {
tokio::task::yield_now().await;
}
assert!(
!dropped.load(Ordering::SeqCst),
"a dropped .spawn() handle does not cancel the task"
);
let tasks = Arc::clone(ctx.coordinator().tasks());
tasks.abort_all();
tasks.wait_idle().await;
assert!(dropped.load(Ordering::SeqCst), "abort_all stops the task");
assert!(
!reached_end.load(Ordering::SeqCst),
"the stopped task does not reach its end"
);
}
#[tokio::test]
async fn straggler_is_aborted_by_timeout_no_succeed_checkpoint() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let sib_started = Arc::new(AtomicBool::new(false));
let sib_completed = Arc::new(AtomicBool::new(false));
let sib_dropped = Arc::new(AtomicBool::new(false));
let ctx_h = ctx.clone();
let s = Arc::clone(&sib_started);
let c = Arc::clone(&sib_completed);
let d = Arc::clone(&sib_dropped);
let result = tokio::time::timeout(
Duration::from_secs(2),
drive_ok(
ctx.clone(),
async move {
let sib = ctx_h
.step(move |_| {
let s = Arc::clone(&s);
let c = Arc::clone(&c);
let d = Arc::clone(&d);
async move {
struct G(Arc<AtomicBool>);
impl Drop for G {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
let _g = G(d);
s.store(true, Ordering::SeqCst);
std::future::pending::<()>().await;
c.store(true, Ordering::SeqCst);
Ok(1i32)
}
})
.name("sibling")
.spawn();
for _ in 0..3 {
tokio::task::yield_now().await;
}
let _ = ctx_h.wait(Duration::from_secs(30)).await;
let _ = sib.await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
),
)
.await;
assert!(
result.is_err(),
"drive_invocation must NOT terminate on its own while a step body \
runs; the Lambda timeout is the backstop"
);
for _ in 0..10 {
tokio::task::yield_now().await;
}
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
sib_started.load(Ordering::SeqCst),
"sibling should have started"
);
assert!(
!sib_completed.load(Ordering::SeqCst),
"sibling must not complete (its body is non-durable pending)"
);
assert!(
sib_dropped.load(Ordering::SeqCst),
"sibling task must be aborted when the timeout drops the handler"
);
let recorded = client.recorded_updates();
assert!(
recorded
.iter()
.all(|u| u.action() != &OperationAction::Succeed),
"no operation should have checkpointed SUCCEED — the straggler \
was aborted before reaching its terminal"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn root_spawn_after_drive_drop_is_stopped() {
let ctx = live_ctx(Arc::new(InMemoryExecutionClient::new(Vec::new())));
let coordinator = Arc::clone(ctx.coordinator());
let (in_poll_tx, in_poll_rx) = std::sync::mpsc::channel::<()>();
let (resume_tx, resume_rx) = std::sync::mpsc::channel::<()>();
let body_started = Arc::new(AtomicBool::new(false));
let body_dropped = Arc::new(AtomicBool::new(false));
let spawned = Arc::new(AtomicBool::new(false));
let ctx_h = ctx.clone();
let started = Arc::clone(&body_started);
let dropped = Arc::clone(&body_dropped);
let after_spawn = Arc::clone(&spawned);
let handler = async move {
in_poll_tx.send(()).expect("the test waits for this signal");
resume_rx.recv().expect("the test resumes the root");
let handle = ctx_h
.step(move |_| {
let started = Arc::clone(&started);
let dropped = Arc::clone(&dropped);
async move {
struct G(Arc<AtomicBool>);
impl Drop for G {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
let _g = G(dropped);
started.store(true, Ordering::SeqCst);
std::future::pending::<()>().await;
Ok(1i32)
}
})
.name("late")
.spawn();
drop(handle);
after_spawn.store(true, Ordering::SeqCst);
std::future::pending::<Result<String, crate::error::WireError>>().await
};
let mut drive = Box::pin(super::drive_invocation(
handler,
crate::task::TaskContext::none(),
Arc::clone(ctx.task_ownership()),
Arc::clone(&coordinator),
ctx.clone(),
None,
));
std::future::poll_fn(|cx| {
assert!(drive.as_mut().poll(cx).is_pending());
std::task::Poll::Ready(())
})
.await;
in_poll_rx
.recv_timeout(Duration::from_secs(5))
.expect("the root is inside its poll");
assert_eq!(coordinator.phase(), super::Phase::Running);
assert!(!coordinator.tasks().is_closed());
drop(drive);
assert_eq!(coordinator.phase(), super::Phase::Finished);
assert!(coordinator.tasks().is_closed());
resume_tx.send(()).expect("the root waits for this signal");
tokio::time::timeout(Duration::from_secs(5), async {
while !spawned.load(Ordering::SeqCst) {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("the root reaches its spawn after the drop");
tokio::time::sleep(Duration::from_millis(50)).await;
tokio::time::timeout(Duration::from_secs(5), coordinator.tasks().wait_idle())
.await
.expect("no operation task outlives the dropped drive");
assert_eq!(coordinator.tasks().live(), 0);
assert!(
!body_started.load(Ordering::SeqCst) || body_dropped.load(Ordering::SeqCst),
"a late operation task that ran was aborted"
);
}
}
#[cfg(test)]
mod scoped_suspension {
use super::InvocationOutcome;
use super::test_support::drive_ok;
use crate::client::{ExecutionClient, InMemoryExecutionClient};
use crate::context::DurableContext;
use crate::engine::CheckpointLog;
use crate::future::Branch;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
fn live_ctx(client: Arc<InMemoryExecutionClient>) -> DurableContext {
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client as Arc<dyn ExecutionClient>,
"token0".to_owned(),
)
}
fn wait_starts(client: &InMemoryExecutionClient) -> usize {
client
.recorded_updates()
.iter()
.filter(|u| u.wait_options().is_some())
.count()
}
#[tokio::test]
async fn sibling_with_runnable_work_completes_before_invocation_suspends() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let b_ran = Arc::new(AtomicBool::new(false));
let b_ran_c = Arc::clone(&b_ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let flag = b_ran_c;
let branches = vec![
Branch::new("A-wait", |cc: DurableContext| {
Box::pin(async move {
cc.wait(Duration::from_secs(30)).await?;
Ok(0_i32)
})
}),
Branch::new("B-step", move |cc: DurableContext| {
Box::pin(async move {
let v = cc.step(|_| async { Ok(7_i32) }).await?;
flag.store(true, Ordering::SeqCst);
Ok(v)
})
}),
];
let _ = ctx_h.parallel(branches).await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
b_ran.load(Ordering::SeqCst),
"sibling B must complete its runnable work before the invocation suspends"
);
}
#[tokio::test]
async fn two_branches_suspend_pending_after_both_parked() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let branches = vec![
Branch::new("a", |cc: DurableContext| {
Box::pin(async move {
cc.wait(Duration::from_secs(30)).await?;
Ok(0_i32)
})
}),
Branch::new("b", |cc: DurableContext| {
Box::pin(async move {
cc.wait(Duration::from_secs(30)).await?;
Ok(0_i32)
})
}),
];
let _ = ctx_h.parallel(branches).await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(
wait_starts(&client),
2,
"both branches must checkpoint WaitStarted before the invocation suspends"
);
}
#[tokio::test]
async fn suspended_branches_retain_slots_peak_bounded_by_cap() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let active = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let ctx_h = ctx.clone();
let active_h = Arc::clone(&active);
let peak_h = Arc::clone(&peak);
let outcome = drive_ok(
ctx.clone(),
async move {
let mut branches = Vec::new();
for i in 0..4 {
let active = Arc::clone(&active_h);
let peak = Arc::clone(&peak_h);
branches.push(Branch::new(format!("w{i}"), move |cc: DurableContext| {
Box::pin(async move {
let cur = active.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(cur, Ordering::SeqCst);
cc.wait(Duration::from_secs(30)).await?;
Ok(0_i32)
})
}));
}
let _ = ctx_h.parallel(branches).max_concurrency(2).await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(
peak.load(Ordering::SeqCst),
2,
"peak concurrent branches must equal the cap — suspended branches hold their slots"
);
assert!(
active.load(Ordering::SeqCst) <= 2,
"no more than cap-many branches may enter while others hold suspended slots"
);
assert_eq!(
wait_starts(&client),
2,
"only the two slot-holding branches start; the other two never do"
);
}
#[tokio::test]
async fn slot_freed_by_completion_lets_next_branch_start() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let second_ran = Arc::new(AtomicBool::new(false));
let second_h = Arc::clone(&second_ran);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let flag = second_h;
let branches = vec![
Branch::new("first", |cc: DurableContext| async move {
let v = cc.step(|_| async { Ok(1_i32) }).await?;
Ok(v)
}),
Branch::new("second", move |cc: DurableContext| {
Box::pin(async move {
let v = cc.step(|_| async { Ok(2_i32) }).await?;
flag.store(true, Ordering::SeqCst);
Ok(v)
})
}),
];
let r = ctx_h.parallel(branches).max_concurrency(1).await;
Ok::<_, crate::error::WireError>(format!("{}", r.is_ok()))
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Complete("true".to_owned()));
assert!(
second_ran.load(Ordering::SeqCst),
"second branch must start after the first frees its slot"
);
}
#[tokio::test]
async fn nested_coordinator_suspension_propagates_to_root_without_hang() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let ctx_h = ctx.clone();
let fut = drive_ok(
ctx.clone(),
async move {
let outer = vec![Branch::new("outer", |cc: DurableContext| {
Box::pin(async move {
let inner = vec![Branch::new("inner-wait", |icc: DurableContext| {
Box::pin(async move {
icc.wait(Duration::from_secs(30)).await?;
Ok(0_i32)
})
})];
cc.parallel(inner)
.await
.map(|v: Vec<i32>| v.into_iter().sum::<i32>())
.map_err(Into::into)
})
})];
let _ = ctx_h.parallel(outer).await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
);
let outcome = tokio::time::timeout(Duration::from_secs(5), fut)
.await
.expect("nested suspension must reach Pending, not hang");
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(
wait_starts(&client),
1,
"the innermost wait must checkpoint WaitStarted"
);
}
#[tokio::test]
async fn min_successful_completion_excludes_parked_branch() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
let branches = vec![
Branch::new("ok", |cc: DurableContext| async move {
let v = cc.step(|_| async { Ok(1_i32) }).await?;
Ok(v)
}),
Branch::new("waiter", |cc: DurableContext| {
Box::pin(async move {
cc.wait(Duration::from_secs(30)).await?;
Ok(2_i32)
})
}),
];
let r = ctx_h
.parallel(branches)
.completion(
crate::builders::map_parallel::CompletionConfig::builder()
.min_successful(1)
.build()
.expect("valid completion config"),
)
.await;
Ok::<_, crate::error::WireError>(format!("ok={}", r.is_ok()))
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Complete("ok=true".to_owned()));
}
#[tokio::test]
async fn root_level_wait_suspends_invocation_immediately() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let after = Arc::new(AtomicBool::new(false));
let after_c = Arc::clone(&after);
let ctx_h = ctx.clone();
let outcome = drive_ok(
ctx.clone(),
async move {
ctx_h.wait(Duration::from_secs(30)).await.map_err(|e| {
crate::error::WireError::new(Some("W".to_owned()), Some(e.to_string()))
})?;
after_c.store(true, Ordering::SeqCst);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
)
.await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
!after.load(Ordering::SeqCst),
"code after a root-level wait must not run"
);
}
}
#[cfg(test)]
mod spawn_scope_regressions {
use super::InvocationOutcome;
use super::test_support::drive_ok;
use crate::client::{ExecutionClient, InMemoryExecutionClient};
use crate::context::DurableContext;
use crate::engine::CheckpointLog;
use aws_sdk_lambda::types::OperationAction;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
const BOUND: Duration = Duration::from_secs(5);
fn live_ctx(client: Arc<InMemoryExecutionClient>) -> DurableContext {
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client as Arc<dyn ExecutionClient>,
"token0".to_owned(),
)
}
fn succeeded(client: &InMemoryExecutionClient, name: &str) -> bool {
client
.recorded_updates()
.iter()
.any(|u| matches!(u.action(), OperationAction::Succeed) && u.name() == Some(name))
}
#[tokio::test]
async fn spawned_step_reaches_terminal_checkpoint_before_pending() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let body_done = Arc::new(AtomicBool::new(false));
let done = Arc::clone(&body_done);
let ctx_h = ctx.clone();
let outcome = tokio::time::timeout(
BOUND,
drive_ok(
ctx.clone(),
async move {
let wait = ctx_h.wait(Duration::from_secs(10)).spawn();
let work = ctx_h
.step(move |_| {
let done = Arc::clone(&done);
async move {
tokio::time::sleep(Duration::from_millis(100)).await;
done.store(true, Ordering::SeqCst);
Ok(7_i32)
}
})
.name("work")
.spawn();
let _ = work.await;
let _ = wait.await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
),
)
.await
.expect("drive_invocation must terminate within the bound, not hang");
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
body_done.load(Ordering::SeqCst),
"the spawned step body must run to completion before the invocation \
suspends; a parked spawned sibling must not abort it"
);
assert!(
succeeded(&client, "work"),
"the spawned step must reach its terminal Succeed checkpoint before \
the invocation reports Pending; otherwise its STARTED checkpoint \
re-executes the body on resume and duplicates side effects"
);
}
#[tokio::test]
async fn joined_spawned_wait_and_step_lets_the_step_finish() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let body_done = Arc::new(AtomicBool::new(false));
let done = Arc::clone(&body_done);
let ctx_h = ctx.clone();
let outcome = tokio::time::timeout(
BOUND,
drive_ok(
ctx.clone(),
async move {
let wait = ctx_h.wait(Duration::from_secs(10)).spawn();
let work = ctx_h
.step(move |_| {
let done = Arc::clone(&done);
async move {
tokio::time::sleep(Duration::from_millis(100)).await;
done.store(true, Ordering::SeqCst);
Ok(7_i32)
}
})
.name("work")
.spawn();
let (_timer, _result) = tokio::join!(wait, work);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
),
)
.await
.expect("drive_invocation must terminate within the bound, not hang");
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
body_done.load(Ordering::SeqCst),
"join! over a spawned wait and a spawned step must not abort the step"
);
assert!(
succeeded(&client, "work"),
"the joined spawned step must reach its terminal Succeed checkpoint \
before the invocation reports Pending"
);
}
#[tokio::test]
async fn spawned_wait_parks_then_owner_does_sequential_step() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let step_done = Arc::new(AtomicBool::new(false));
let done = Arc::clone(&step_done);
let ctx_h = ctx.clone();
let outcome = tokio::time::timeout(
BOUND,
drive_ok(
ctx.clone(),
async move {
let _wait = ctx_h.wait(Duration::from_secs(10)).spawn();
ctx_h.hold_activity(tokio::task::yield_now()).await;
let _result = ctx_h
.step(move |_| {
let done = Arc::clone(&done);
async move {
done.store(true, Ordering::SeqCst);
Ok(42_i32)
}
})
.name("sequential_work")
.await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
),
)
.await
.expect("drive_invocation must terminate within the bound, not hang");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(
step_done.load(Ordering::SeqCst),
"the non-spawned step must complete before Pending; a parked spawn \
must not pre-empt the owner's sequential work"
);
assert!(
succeeded(&client, "sequential_work"),
"the sequential step must reach its terminal Succeed checkpoint \
before the invocation reports Pending"
);
}
#[tokio::test]
async fn spawned_combinator_with_parking_input_does_not_deadlock() {
let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
let ctx = live_ctx(Arc::clone(&client));
let signal = Arc::clone(ctx.coordinator());
let sibling_done = Arc::new(AtomicBool::new(false));
let done = Arc::clone(&sibling_done);
let ctx_h = ctx.clone();
let outcome = tokio::time::timeout(
BOUND,
drive_ok(
ctx.clone(),
async move {
let parking_wait = ctx_h.wait(Duration::from_mins(1)).spawn();
let instant_step = ctx_h.step(|_| async { Ok(()) }).name("inner").future();
let combo = ctx_h.try_join_all([parking_wait, instant_step]).spawn();
let sibling = ctx_h
.step(move |_| {
let d = Arc::clone(&done);
async move {
tokio::time::sleep(Duration::from_millis(100)).await;
d.store(true, Ordering::SeqCst);
Ok(42_i32)
}
})
.name("sibling")
.spawn();
let _ = tokio::join!(combo, sibling);
Ok::<_, crate::error::WireError>("done".to_owned())
},
Arc::clone(ctx.task_ownership()),
signal,
),
)
.await
.expect(
"drive_invocation must terminate within the bound; \
a deadlock from the spawned combinator defect would hang here",
);
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
sibling_done.load(Ordering::SeqCst),
"the runnable sibling must complete before the invocation \
suspends; the combinator's suspension must not abort it"
);
assert!(
succeeded(&client, "sibling"),
"the sibling must reach its terminal Succeed checkpoint \
before the invocation reports Pending"
);
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod task_gateway {
use super::{InvocationOutcome, drive_outcome};
use crate::client::{ExecutionClient, InMemoryExecutionClient};
use crate::context::DurableContext;
use crate::engine::CheckpointLog;
use crate::future::DurableFuture;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
fn live_ctx() -> DurableContext {
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::new(InMemoryExecutionClient::new(Vec::new())) as Arc<dyn ExecutionClient>,
"token0".to_owned(),
)
}
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
fn spawn_holding_step(ctx: &DurableContext, dropped: &Arc<AtomicBool>) -> DurableFuture<()> {
let dropped = Arc::clone(dropped);
ctx.step(move |_| {
let dropped = Arc::clone(&dropped);
async move {
let _guard = DropFlag(dropped);
std::future::pending::<()>().await;
Ok(())
}
})
.name("hold")
.spawn()
}
async fn yield_three_times() {
for _ in 0..3 {
tokio::task::yield_now().await;
}
}
#[tokio::test]
async fn aborted_spawned_body_drops_before_invocation_returns() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let dropped = Arc::new(AtomicBool::new(false));
let handler_dropped = Arc::clone(&dropped);
let outcome = drive_outcome(
async move {
let _wait = ctx.wait(Duration::from_secs(10)).name("timer").spawn();
let _step = spawn_holding_step(&ctx, &handler_dropped);
yield_three_times().await;
Ok::<_, crate::error::WireError>("done".to_owned())
},
ownership,
signal,
root,
None,
)
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(
dropped.load(Ordering::SeqCst),
"the step body must be dropped before the invocation returns"
);
}
#[tokio::test]
async fn handler_panic_returns_payload_after_cleanup() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let dropped = Arc::new(AtomicBool::new(false));
let handler_dropped = Arc::clone(&dropped);
let outcome = drive_outcome(
async move {
let _step = spawn_holding_step(&ctx, &handler_dropped);
yield_three_times().await;
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await;
let Err(payload) = outcome else {
panic!("expected the handler panic, got {outcome:?}");
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"handler boom"));
assert!(dropped.load(Ordering::SeqCst), "the step body was dropped");
assert_eq!(tasks.live(), 0);
}
#[tokio::test]
async fn handler_panic_aborts_task_whose_handle_left_the_handler() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let dropped = Arc::new(AtomicBool::new(false));
let handler_dropped = Arc::clone(&dropped);
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let handler_escaped = Arc::clone(&escaped);
let outcome = drive_outcome(
async move {
let step = spawn_holding_step(&ctx, &handler_dropped);
*handler_escaped.lock().expect("lock") = Some(step);
yield_three_times().await;
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await;
assert!(outcome.is_err(), "expected the handler panic");
assert!(
escaped.lock().expect("lock").is_some(),
"the test still holds the handle"
);
assert!(dropped.load(Ordering::SeqCst), "the step body was dropped");
assert_eq!(tasks.live(), 0);
}
#[tokio::test]
async fn handler_panic_with_parked_spawn_returns_payload() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let parked_signal = Arc::clone(&signal);
let outcome = drive_outcome(
async move {
let _wait = ctx.wait(Duration::from_secs(10)).name("timer").spawn();
ctx.hold_activity(async {
for _ in 0..1000 {
if parked_signal.live_waits() == 1 {
break;
}
tokio::task::yield_now().await;
}
})
.await;
assert_eq!(parked_signal.live_waits(), 1, "the wait is registered");
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await;
let Err(payload) = outcome else {
panic!("expected the handler panic, got {outcome:?}");
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"handler boom"));
assert_eq!(tasks.live(), 0);
}
#[tokio::test]
async fn handler_panic_after_suspend_request_returns_payload() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let handler_signal = Arc::clone(&signal);
let outcome = drive_outcome(
async move {
let _ctx = ctx;
let _registration =
handler_signal.register_test_wait("op", std::task::Waker::noop());
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await;
let Err(payload) = outcome else {
panic!("expected the handler panic, got {outcome:?}");
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"handler boom"));
assert_eq!(tasks.live(), 0);
}
#[tokio::test]
async fn fatal_error_wins_over_handler_panic() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let handler_signal = Arc::clone(&signal);
let outcome = drive_outcome(
async move {
let _ctx = ctx;
handler_signal.record_fatal("Fatal".to_owned(), "mismatch".to_owned());
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await
.expect("the fatal error takes precedence over the panic");
assert_eq!(
outcome,
InvocationOutcome::Failed {
error: crate::error::WireError::new(
Some("Fatal".to_owned()),
Some("mismatch".to_owned()),
),
}
);
}
pub(super) struct NestedPanicPayload;
impl Drop for NestedPanicPayload {
fn drop(&mut self) {
std::panic::panic_any(InnerPanicPayload);
}
}
pub(super) struct InnerPanicPayload;
impl Drop for InnerPanicPayload {
fn drop(&mut self) {
panic!("inner payload drop");
}
}
struct FatalOnDrop(Arc<crate::coordinator::InvocationCoordinator>);
impl Drop for FatalOnDrop {
fn drop(&mut self) {
self.0
.record_fatal("Fatal".to_owned(), "mismatch during cleanup".to_owned());
}
}
#[tokio::test(flavor = "current_thread")]
async fn fatal_recorded_during_panic_cleanup_wins_over_panic() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let body_signal = Arc::clone(&signal);
let started = Arc::new(AtomicBool::new(false));
let body_started = Arc::clone(&started);
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let handler_escaped = Arc::clone(&escaped);
let check_signal = Arc::clone(&signal);
let outcome = drive_outcome(
async move {
let step = ctx
.step(move |_| {
let guard = FatalOnDrop(Arc::clone(&body_signal));
let started = Arc::clone(&body_started);
async move {
let _guard = guard;
started.store(true, Ordering::SeqCst);
std::future::pending::<()>().await;
Ok(())
}
})
.name("hold")
.spawn();
*handler_escaped.lock().expect("lock") = Some(step);
yield_three_times().await;
assert!(check_signal.fatal_error().is_none(), "no fatal yet");
panic!("handler boom");
},
ownership,
signal,
root,
None,
)
.await
.expect("the fatal error recorded during cleanup takes precedence over the panic");
assert!(started.load(Ordering::SeqCst), "the step body started");
assert_eq!(
outcome,
InvocationOutcome::Failed {
error: crate::error::WireError::new(
Some("Fatal".to_owned()),
Some("mismatch during cleanup".to_owned()),
),
}
);
assert_eq!(tasks.live(), 0);
}
struct OnDrop(
Arc<crate::coordinator::InvocationCoordinator>,
Arc<dyn Fn(&crate::coordinator::InvocationCoordinator) + Send + Sync>,
);
impl Drop for OnDrop {
fn drop(&mut self) {
(self.1)(&self.0);
}
}
async fn drive_nested_handler_panic_with_cleanup(
on_cleanup: Arc<dyn Fn(&crate::coordinator::InvocationCoordinator) + Send + Sync>,
) -> (
Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>,
usize,
) {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let body_signal = Arc::clone(&signal);
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let handler_escaped = Arc::clone(&escaped);
let outcome = drive_outcome(
async move {
let step = ctx
.step(move |_| {
let guard = OnDrop(Arc::clone(&body_signal), Arc::clone(&on_cleanup));
async move {
let _guard = guard;
std::future::pending::<()>().await;
Ok(())
}
})
.name("hold")
.spawn();
*handler_escaped.lock().expect("lock") = Some(step);
yield_three_times().await;
std::panic::panic_any(NestedPanicPayload);
},
ownership,
signal,
root,
None,
)
.await;
(outcome, tasks.live())
}
#[tokio::test(flavor = "current_thread")]
async fn fatal_during_cleanup_wins_over_nested_panicking_handler_payload() {
let (outcome, live) = drive_nested_handler_panic_with_cleanup(Arc::new(|coordinator| {
coordinator.record_fatal("Fatal".to_owned(), "during cleanup".to_owned());
}))
.await;
let Ok(InvocationOutcome::Failed { error }) = &outcome else {
panic!("expected the fatal outcome, got {outcome:?}");
};
assert_eq!(error.error_message(), Some("during cleanup"));
assert_eq!(live, 0);
}
#[tokio::test]
async fn handler_value_with_live_registration_completes() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let handler_signal = Arc::clone(&signal);
let outcome = drive_outcome(
async move {
let _ctx = ctx;
let _registration =
handler_signal.register_test_wait("op", std::task::Waker::noop());
Ok::<_, crate::error::WireError>("done".to_owned())
},
ownership,
signal,
root,
None,
)
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
}
#[tokio::test]
async fn handler_runs_as_blessed_operation_task() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let outcome = drive_outcome(
async move {
let on_task = tokio::task::try_id().is_some();
let value = ctx
.step(|_| async { Ok(1_i32) })
.name("s")
.await
.map_err(|e| {
crate::error::WireError::new(None::<String>, Some(e.to_string()))
})?;
Ok::<_, crate::error::WireError>(format!("{on_task}:{value}"))
},
Arc::clone(&ownership),
signal,
root,
None,
)
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("true:1".to_owned()));
assert_eq!(ownership.blessed_count(), 0);
}
#[tokio::test]
async fn wait_idle_in_handler_does_not_wait_for_handler() {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let signal = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let tasks = Arc::clone(signal.tasks());
let outcome = drive_outcome(
async move {
let step = ctx.step(|_| async { Ok(1_i32) }).name("quick").spawn();
let _ = step.await;
let idle = tokio::time::timeout(Duration::from_secs(1), tasks.wait_idle()).await;
Ok::<_, crate::error::WireError>(idle.is_ok().to_string())
},
ownership,
signal,
root,
None,
)
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("true".to_owned()));
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod coordinator_suspension {
use super::test_support::{
EventMessages, ScriptedClient, drive_handler, named, operation_like, scripted_ctx, wire_err,
};
use super::{InvocationOutcome, UNTRACKED_DEADLINE_WARNING};
use crate::coordinator::{PermitKind, Phase};
use crate::engine::CheckpointStatus;
use aws_sdk_lambda::types::{OperationAction, OperationStatus};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
async fn until(ctx: &crate::DurableContext, condition: impl Fn() -> bool) -> bool {
ctx.hold_activity(async {
for _ in 0..1000 {
if condition() {
return true;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
false
})
.await
}
fn completes_on_step_succeed(
step: &'static str,
waiting: &'static str,
status: OperationStatus,
) -> ScriptedClient {
ScriptedClient::new(move |updates, history| {
if named(updates, step, &OperationAction::Succeed).is_none() {
return Vec::new();
}
named(history, waiting, &OperationAction::Start)
.map(|start| {
operation_like(start, status.clone())
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
})
}
async fn after_start_of(
client: &Arc<ScriptedClient>,
waiting: &str,
) -> Result<u32, crate::BoxError> {
for _ in 0..1000 {
if client.received(waiting, &OperationAction::Start) {
return Ok(2);
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
Err("the waiting operation never started".into())
}
#[tokio::test(start_paused = true)]
async fn lone_wait_suspends_at_once() {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
ctx.wait(Duration::from_secs(1)).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(client.call_count(), 1);
}
#[tokio::test(start_paused = true)]
async fn untracked_await_does_not_suspend_and_warns() {
let (capture, _guard) = EventMessages::install();
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let drive = drive_handler(ctx, Some(Duration::from_secs(2)), |_ctx| async move {
tokio::time::sleep(Duration::from_secs(3600)).await;
Ok("done".to_owned())
});
let timed = tokio::time::timeout(Duration::from_secs(5), drive).await;
assert!(timed.is_err(), "the untracked await does not suspend");
assert_eq!(
capture.at(tracing::Level::WARN),
vec![UNTRACKED_DEADLINE_WARNING.to_owned()]
);
}
#[tokio::test(start_paused = true)]
async fn deadline_during_tracked_work_does_not_warn() {
let (capture, _guard) = EventMessages::install();
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let outcome = drive_handler(ctx, Some(Duration::from_secs(1)), |ctx| async move {
ctx.step(|_| async {
tokio::time::sleep(Duration::from_secs(2)).await;
Ok(1_u32)
})
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_eq!(capture.at(tracing::Level::WARN), Vec::<String>::new());
}
#[tokio::test(start_paused = true)]
async fn sibling_response_completes_wait_in_invocation() {
let (events, _guard) = EventMessages::install();
let client = Arc::new(completes_on_step_succeed(
"s",
"w",
OperationStatus::Succeeded,
));
let ctx = scripted_ctx(&client);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(1)).name("w").spawn();
ctx.step(move |_| async move { after_start_of(&body_client, "w").await })
.name("s")
.await
.map_err(wire_err)?;
w.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_eq!(events.replayed(), 0, "a resumed wait emits no replay event");
}
#[tokio::test(start_paused = true)]
async fn buffered_checkpoint_holds_permit_until_merged() {
let client = Arc::new(completes_on_step_succeed(
"s",
"w",
OperationStatus::Succeeded,
));
let ctx = crate::DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::from_millis(50)),
);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(1)).name("w").spawn();
ctx.step(move |_| async move { after_start_of(&body_client, "w").await })
.name("s")
.await
.map_err(wire_err)?;
w.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
}
fn hour_window_ctx(
client: &Arc<ScriptedClient>,
log: crate::engine::CheckpointLog,
) -> crate::DurableContext {
crate::DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(log),
Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::from_secs(3600)),
)
}
#[tokio::test(start_paused = true)]
async fn completion_preempts_window_of_dropped_contributor() {
let client = Arc::new(ScriptedClient::empty());
let ctx = hour_window_ctx(&client, crate::engine::CheckpointLog::empty());
let coordinator = Arc::clone(ctx.coordinator());
let started = tokio::time::Instant::now();
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(10)).name("w").spawn();
let buffered = ctx
.hold_activity(async {
for _ in 0..1000 {
if ctx.coordinator().permits(PermitKind::Checkpoint) == 1 {
return true;
}
tokio::task::yield_now().await;
}
false
})
.await;
if !buffered {
return Err(crate::error::WireError::new(
None::<String>,
Some("the START of w was never buffered"),
));
}
drop(w);
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(
started.elapsed() < Duration::from_secs(60),
"completion waited {:?} of the window",
started.elapsed()
);
assert!(
client.received("w", &OperationAction::Start),
"the buffered START is flushed"
);
assert_eq!(coordinator.tasks().live(), 0);
}
#[tokio::test(start_paused = true)]
async fn waiting_operation_preempts_window() {
let client = Arc::new(ScriptedClient::empty());
let wait_id = crate::engine::compute_wire_id_public("1");
let log = crate::engine::CheckpointLog::from_records(vec![(
wait_id.clone(),
crate::engine::CheckpointRecord::with_status(&wait_id, CheckpointStatus::Started),
)]);
let ctx = hour_window_ctx(&client, log);
let started = tokio::time::Instant::now();
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(10)).name("w").spawn();
ctx.step(|_| async { Ok(1_u32) })
.name("s")
.await
.map_err(wire_err)?;
w.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
started.elapsed() < Duration::from_secs(60),
"the step's writes waited {:?} of the window",
started.elapsed()
);
assert!(client.received("s", &OperationAction::Succeed));
}
#[tokio::test(start_paused = true)]
async fn wait_resumed_maps_terminal_status() {
let client = Arc::new(completes_on_step_succeed("s", "w", OperationStatus::Failed));
let ctx = scripted_ctx(&client);
let body_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(1)).name("w").spawn();
ctx.step(move |_| async move { after_start_of(&body_client, "w").await })
.name("s")
.await
.map_err(wire_err)?;
let err = w.await.expect_err("the wait record failed");
let crate::error::OperationErrorKind::Wait(wait_err) = err.kind() else {
panic!("expected a wait error, got {err:?}");
};
let crate::error::WaitErrorKind::UnexpectedStatus(status) = wait_err.kind();
Ok(status.status().to_owned())
})
.await
.expect("the handler does not panic");
assert_eq!(outcome, InvocationOutcome::Complete("FAILED".to_owned()));
}
#[tokio::test(start_paused = true)]
async fn resolved_handler_completes_with_dropped_spawned_wait() {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let handler_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(Duration::from_secs(10)).name("w").spawn();
let registered = ctx
.hold_activity(async {
for _ in 0..1000 {
if handler_client.received("w", &OperationAction::Start)
&& ctx.coordinator().live_waits() == 1
{
return true;
}
tokio::task::yield_now().await;
}
false
})
.await;
if !registered {
return Err(crate::error::WireError::new(
None::<String>,
Some("the wait never registered"),
));
}
drop(w);
Ok(serde_json::to_string("done").expect("a string serializes"))
})
.await;
assert!(
matches!(&outcome, Ok(InvocationOutcome::Complete(v)) if v == "\"done\""),
"unexpected outcome: {outcome:?}"
);
let w_updates: Vec<_> = client
.updates()
.into_iter()
.filter(|u| u.name() == Some("w"))
.collect();
assert_eq!(w_updates.len(), 1, "only the START of w was written");
assert_eq!(
w_updates
.first()
.map(aws_sdk_lambda::types::OperationUpdate::action),
Some(&OperationAction::Start)
);
assert_eq!(coordinator.tasks().live(), 0);
assert_eq!(coordinator.live_waits(), 0);
}
fn assert_clean(coordinator: &crate::coordinator::InvocationCoordinator) {
assert_eq!(coordinator.permits(PermitKind::Activity), 0);
assert_eq!(coordinator.permits(PermitKind::Checkpoint), 0);
assert_eq!(coordinator.live_waits(), 0);
assert_eq!(coordinator.tasks().live(), 0);
}
fn spawn_wait_and_stuck_step(
ctx: &crate::DurableContext,
) -> (crate::DurableFuture<()>, crate::DurableFuture<()>) {
let wait = ctx.wait(Duration::from_secs(3600)).name("w").spawn();
let step = ctx
.step(|_| async {
std::future::pending::<()>().await;
Ok(())
})
.name("stuck")
.spawn();
(wait, step)
}
#[tokio::test(start_paused = true)]
async fn invocation_end_leaves_no_task_permit_or_registration() {
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let _ops = spawn_wait_and_stuck_step(&ctx);
ctx.hold_activity(tokio::time::sleep(Duration::from_millis(10)))
.await;
Ok("a".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("a".to_owned()));
assert_clean(&coordinator);
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
drop(spawn_wait_and_stuck_step(&ctx));
ctx.hold_activity(tokio::time::sleep(Duration::from_millis(10)))
.await;
Ok("b".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("b".to_owned()));
assert_clean(&coordinator);
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let _ops = spawn_wait_and_stuck_step(&ctx);
ctx.hold_activity(tokio::time::sleep(Duration::from_millis(10)))
.await;
Err(crate::error::WireError::new(Some("E"), Some("c")))
})
.await
.expect("no panic");
assert!(matches!(outcome, InvocationOutcome::Failed { .. }));
assert_clean(&coordinator);
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let _ops = spawn_wait_and_stuck_step(&ctx);
ctx.hold_activity(tokio::time::sleep(Duration::from_millis(10)))
.await;
panic!("d");
})
.await;
let Err(payload) = outcome else {
panic!("expected the handler panic, got {outcome:?}");
};
assert_eq!(payload.downcast_ref::<&str>(), Some(&"d"));
assert_clean(&coordinator);
}
#[tokio::test(start_paused = true)]
async fn unawaited_finished_spawn_does_not_block_suspension() {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let start = tokio::time::Instant::now();
let handler_client = Arc::clone(&client);
let outcome = drive_handler(ctx, Some(Duration::from_mins(15)), |ctx| async move {
let _h = ctx.step(|_| async { Ok(1_u32) }).name("h").spawn();
let tasks = Arc::clone(ctx.coordinator().tasks());
let exited = until(&ctx, || {
handler_client.received("h", &OperationAction::Succeed) && tasks.live() == 0
})
.await;
if !exited {
return Err(crate::error::WireError::new(
None::<String>,
Some("h never ended"),
));
}
ctx.wait(Duration::from_secs(3600))
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Pending);
assert!(
start.elapsed() < Duration::from_secs(60),
"the invocation suspended at once, not at the deadline"
);
}
#[test]
fn exhausted_budget_does_not_suspend_runnable_handler() {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
runtime.block_on(async {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let _wait = ctx.wait(Duration::from_secs(3600)).name("w").spawn();
let coordinator = Arc::clone(ctx.coordinator());
let registered = until(&ctx, || coordinator.live_waits() == 1).await;
if !registered {
return Err(crate::error::WireError::new(
None::<String>,
Some("no wait"),
));
}
let sibling = ctx.step(|_| async { Ok(1_u32) }).name("sibling").spawn();
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<u32>();
for value in 0..20_000 {
tx.send(value).expect("the receiver is alive");
}
drop(tx);
let mut received = 0_u32;
while rx.recv().await.is_some() {
received += 1;
}
sibling.await.map_err(wire_err)?;
ctx.step(|_| async { Ok(2_u32) })
.name("after")
.await
.map_err(wire_err)?;
Ok(received.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("20000".to_owned()));
assert!(client.received("after", &OperationAction::Succeed));
});
}
#[tokio::test(start_paused = true)]
async fn detached_write_in_flight_at_finish_completes() {
let client = Arc::new(
ScriptedClient::new(|updates, _| {
named(updates, "quick", &OperationAction::Succeed)
.map(|succeed| {
operation_like(succeed, OperationStatus::Succeeded)
.build()
.expect("all required fields set")
})
.into_iter()
.collect()
})
.with_delay(Duration::from_millis(500)),
);
let ctx = scripted_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let log_ctx = ctx.clone();
let handler_client = Arc::clone(&client);
let start = tokio::time::Instant::now();
let outcome = drive_handler(ctx, None, |ctx| async move {
let h = ctx.step(|_| async { Ok(1_u32) }).name("quick").spawn();
let sent = until(&ctx, || {
handler_client.received("quick", &OperationAction::Succeed)
})
.await;
if !sent {
return Err(crate::error::WireError::new(
None::<String>,
Some("no SUCCEED"),
));
}
drop(h);
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert!(start.elapsed() >= Duration::from_millis(500));
assert_eq!(
log_ctx.checkpoint_status_view("1").map(|v| v.status),
Some(CheckpointStatus::Succeeded),
"the in-flight write's response was merged"
);
assert_eq!(coordinator.tasks().live(), 0);
}
async fn yield_with_live_wait() -> (InvocationOutcome, bool) {
let client = Arc::new(ScriptedClient::empty());
let ctx = scripted_ctx(&client);
let after_ran = Arc::new(AtomicBool::new(false));
let handler_ran = Arc::clone(&after_ran);
let outcome = drive_handler(ctx, None, |ctx| async move {
let _wait = ctx.wait(Duration::from_secs(3600)).name("w").spawn();
let coordinator = Arc::clone(ctx.coordinator());
let ready = ctx
.hold_activity(async {
for _ in 0..10_000 {
if coordinator.live_waits() == 1
&& coordinator.permits(PermitKind::Activity) == 2
&& coordinator.permits(PermitKind::Checkpoint) == 0
{
return true;
}
tokio::task::yield_now().await;
}
false
})
.await;
if !ready {
return Err(crate::error::WireError::new(
None::<String>,
Some("not ready"),
));
}
tokio::task::yield_now().await;
ctx.step(move |_| async move {
handler_ran.store(true, Ordering::SeqCst);
Ok(1_u32)
})
.name("after")
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
(outcome, after_ran.load(Ordering::SeqCst))
}
#[test]
fn root_yield_now_is_a_boundary_on_every_runtime() {
let current = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
let multi = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.enable_all()
.build()
.expect("runtime");
for _ in 0..100 {
let (outcome, after_ran) = current.block_on(yield_with_live_wait());
assert_eq!(
outcome,
InvocationOutcome::Pending,
"current-thread runtime"
);
assert!(!after_ran, "current-thread runtime");
let (outcome, after_ran) = multi.block_on(yield_with_live_wait());
assert_eq!(outcome, InvocationOutcome::Pending, "multi-thread block_on");
assert!(!after_ran, "multi-thread block_on");
let (outcome, after_ran) = multi
.block_on(async { spawned(yield_with_live_wait()).await })
.expect("the spawned scenario does not panic");
assert_eq!(outcome, InvocationOutcome::Pending, "multi-thread task");
assert!(!after_ran, "multi-thread task");
}
}
#[expect(clippy::disallowed_methods)]
fn spawned<F>(future: F) -> tokio::task::JoinHandle<F::Output>
where
F: Future + Send + 'static,
F::Output: Send + 'static,
{
tokio::spawn(future)
}
#[tokio::test(start_paused = true)]
async fn phase_is_finished_after_every_drive() {
let ctx = scripted_ctx(&Arc::new(ScriptedClient::empty()));
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |_ctx| async move { Ok("done".to_owned()) })
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_eq!(coordinator.phase(), Phase::Finished);
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod refresh_scheduler {
use super::InvocationOutcome;
use super::test_support::{
RefreshClient, RefreshRequest, assert_near, assert_times, callback_record, callback_starts,
drive_handler, named, operation_like, poll_once, refresh_ctx, wire_err,
};
use crate::BoxError;
use crate::builders::wait_for_condition::WaitDecision;
use crate::client::ExecutionClientError;
use crate::coordinator::PermitKind;
use crate::engine::CheckpointStatus;
use aws_sdk_lambda::types::{
ChainedInvokeDetails, Operation, OperationAction, OperationStatus, OperationUpdate,
StepDetails, WaitDetails,
};
use std::sync::{Arc, Mutex};
use std::time::Duration;
const fn secs(s: u64) -> Duration {
Duration::from_secs(s)
}
const fn millis(ms: u64) -> Duration {
Duration::from_millis(ms)
}
fn wait_record(
req: &RefreshRequest<'_>,
start: &OperationUpdate,
status: OperationStatus,
end_in: Option<Duration>,
) -> Operation {
let mut builder = operation_like(start, status);
if let Some(end_in) = end_in {
builder = builder.wait_details(
WaitDetails::builder()
.scheduled_end_timestamp(req.timestamp_in(end_in))
.build(),
);
}
builder.build().expect("all required fields set")
}
fn invoke_record(
start: &OperationUpdate,
status: OperationStatus,
result: Option<&str>,
) -> Operation {
let mut details = ChainedInvokeDetails::builder();
if let Some(result) = result {
details = details.result(result);
}
operation_like(start, status)
.chained_invoke_details(details.build())
.build()
.expect("all required fields set")
}
fn condition_record(
req: &RefreshRequest<'_>,
update: &OperationUpdate,
status: OperationStatus,
payload: Option<&str>,
next_in: Option<Duration>,
) -> Operation {
let mut details = StepDetails::builder().attempt(1);
if let Some(payload) = payload {
details = details.result(payload);
}
if let Some(next_in) = next_in {
details = details.next_attempt_timestamp(req.timestamp_in(next_in));
}
operation_like(update, status)
.step_details(details.build())
.build()
.expect("all required fields set")
}
fn this<'a>(
req: &'a RefreshRequest<'_>,
name: &str,
action: &OperationAction,
) -> Option<&'a OperationUpdate> {
named(req.updates, name, action)
}
fn ever<'a>(
req: &'a RefreshRequest<'_>,
name: &str,
action: &OperationAction,
) -> Option<&'a OperationUpdate> {
named(req.history, name, action)
}
fn sleeping_step(
ctx: &crate::DurableContext,
name: &str,
d: Duration,
) -> crate::DurableFuture<u32> {
ctx.step(move |_| async move {
tokio::time::sleep(d).await;
Ok(1_u32)
})
.name(name)
.future()
}
fn spawned_sleeping_step(
ctx: &crate::DurableContext,
name: &str,
d: Duration,
) -> crate::DurableFuture<u32> {
ctx.step(move |_| async move {
tokio::time::sleep(d).await;
Ok(1_u32)
})
.name(name)
.spawn()
}
async fn parallel_wait_then_step(ctx: crate::DurableContext) -> Result<String, BoxError> {
let branches: Vec<crate::Branch<u32>> = vec![
crate::Branch::new("A", |child: crate::DurableContext| async move {
child.wait(secs(5)).name("w").await?;
let a = child.step(|_| async { Ok(1_u32) }).name("a").await?;
Ok(a)
}),
crate::Branch::new("B", |child: crate::DurableContext| async move {
let b = child
.step(|_| async {
tokio::time::sleep(secs(60)).await;
Ok(2_u32)
})
.name("b")
.await?;
Ok(b)
}),
];
let results = ctx.parallel(branches).name("p").await?;
Ok(format!("{results:?}"))
}
fn test3_client() -> RefreshClient {
RefreshClient::new(|req| {
let mut ops: Vec<Operation> = req
.updates
.iter()
.map(|u| {
let status = match u.action() {
OperationAction::Succeed => OperationStatus::Succeeded,
OperationAction::Fail => OperationStatus::Failed,
_ => OperationStatus::Started,
};
let end_in = (u.name() == Some("w")).then_some(secs(5));
wait_record(req, u, status, end_in)
})
.collect();
if req.empty_number == 1
&& let Some(start) = ever(req, "w", &OperationAction::Start)
{
ops.push(wait_record(req, start, OperationStatus::Succeeded, None));
}
Ok(ops)
})
}
#[tokio::test(start_paused = true)]
async fn parallel_wait_then_step_finishes_in_one_invocation() {
let client = Arc::new(test3_client());
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
parallel_wait_then_step(ctx)
.await
.map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("[1, 2]".to_owned()));
assert_times(&client.empty_calls(), &[secs(5)]);
assert_near(
client
.time_of("a", &OperationAction::Start)
.expect("a started"),
secs(5),
);
}
async fn callback_beside_step(
client: &Arc<RefreshClient>,
remaining: Option<Duration>,
step: Duration,
) -> InvocationOutcome {
let ctx = refresh_ctx(client);
drive_handler(ctx, remaining, move |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
sleeping_step(&ctx, "s", step).await.map_err(wire_err)?;
h.await.map_err(wire_err)
})
.await
.expect("no panic")
}
fn callback_client(complete: bool) -> RefreshClient {
RefreshClient::new(move |req| {
let mut ops = callback_starts(req.updates);
if complete
&& req.empty_number == 1
&& let Some(start) = ever(req, "cb", &OperationAction::Start)
{
ops.push(callback_record(
start,
OperationStatus::Succeeded,
Some("\"ok\""),
));
}
Ok(ops)
})
}
#[tokio::test(start_paused = true)]
async fn callback_refresh_continues_in_invocation() {
let client = Arc::new(callback_client(true));
let outcome = callback_beside_step(&client, None, secs(3)).await;
assert_eq!(outcome, InvocationOutcome::Complete("ok".to_owned()));
assert_times(&client.empty_calls(), &[secs(1)]);
}
async fn invoke_beside_step(client: &Arc<RefreshClient>) -> InvocationOutcome {
let ctx = refresh_ctx(client);
drive_handler(ctx, None, |ctx| async move {
let mut h = std::pin::pin!(ctx.invoke::<u32>("target", 1_u32).name("inv").spawn());
poll_once(h.as_mut()).await;
sleeping_step(&ctx, "s", secs(5)).await.map_err(wire_err)?;
let value = h.await.map_err(wire_err)?;
Ok(value.to_string())
})
.await
.expect("no panic")
}
#[tokio::test(start_paused = true)]
async fn invoke_refresh_backs_off() {
let client = Arc::new(RefreshClient::new(|req| {
let Some(start) = ever(req, "inv", &OperationAction::Start) else {
return Ok(Vec::new());
};
Ok(match req.empty_number {
1 | 2 => vec![invoke_record(start, OperationStatus::Started, None)],
3 => vec![invoke_record(start, OperationStatus::Succeeded, Some("7"))],
_ => Vec::new(),
})
}));
let outcome = invoke_beside_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
assert_times(&client.empty_calls(), &[secs(1), secs(2), secs(4)]);
}
#[tokio::test(start_paused = true)]
async fn changed_status_restarts_schedule() {
let client = Arc::new(RefreshClient::new(|req| {
let Some(start) = ever(req, "inv", &OperationAction::Start) else {
return Ok(Vec::new());
};
Ok(match req.empty_number {
1 => vec![invoke_record(start, OperationStatus::Pending, None)],
2 => vec![invoke_record(start, OperationStatus::Succeeded, Some("7"))],
_ => Vec::new(),
})
}));
let outcome = invoke_beside_step(&client).await;
assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
assert_times(&client.empty_calls(), &[secs(1), secs(2)]);
}
type CheckFuture = std::future::Ready<Result<i32, BoxError>>;
fn condition(
ctx: &crate::DurableContext,
start: tokio::time::Instant,
checks: &Arc<Mutex<Vec<Duration>>>,
) -> crate::builders::WaitForConditionBuilder<
i32,
impl Fn(crate::StepContext, i32) -> CheckFuture + Send + Sync + 'static,
CheckFuture,
crate::serdes::JsonSerdes,
crate::builders::wait_for_condition::WithWaitStrategy<i32>,
> {
let checks = Arc::clone(checks);
ctx.wait_for_condition(
move |_ctx, state: i32| {
checks.lock().expect("lock").push(start.elapsed());
std::future::ready(Ok::<i32, BoxError>(state + 1))
},
0,
)
.wait_strategy_fn(|state: i32, _attempt| {
if state >= 2 {
WaitDecision::complete()
} else {
WaitDecision::continue_with(secs(1))
}
})
.name("cond")
}
fn condition_client(ready_on: usize) -> RefreshClient {
RefreshClient::new(move |req| {
let mut ops = Vec::new();
if let Some(retry) = this(req, "cond", &OperationAction::Retry) {
ops.push(condition_record(
req,
retry,
OperationStatus::Pending,
retry.payload(),
Some(secs(1)),
));
}
if req.empty_number == ready_on
&& let Some(retry) = ever(req, "cond", &OperationAction::Retry)
{
ops.push(condition_record(
req,
retry,
OperationStatus::Ready,
retry.payload(),
None,
));
}
Ok(ops)
})
}
#[tokio::test(start_paused = true)]
async fn condition_check_refreshes_at_next_attempt() {
let client = Arc::new(condition_client(1));
let ctx = refresh_ctx(&client);
let start = tokio::time::Instant::now();
let checks = Arc::new(Mutex::new(Vec::new()));
let handler_checks = Arc::clone(&checks);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let mut h = std::pin::pin!(condition(&ctx, start, &handler_checks).spawn());
poll_once(h.as_mut()).await;
sleeping_step(&ctx, "s", secs(3)).await.map_err(wire_err)?;
let state = h.await.map_err(wire_err)?;
Ok(state.to_string())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("2".to_owned()));
let checks = checks.lock().expect("lock").clone();
assert_times(&checks, &[secs(0), secs(1)]);
}
#[tokio::test(start_paused = true)]
async fn refresh_wakes_ready_operations_in_sorted_order() {
let end = std::time::SystemTime::now() + secs(1);
let client = Arc::new(RefreshClient::new(move |req| {
let mut ops = Vec::new();
for name in ["w1", "w2"] {
if let Some(start) = this(req, name, &OperationAction::Start) {
ops.push(
operation_like(start, OperationStatus::Started)
.wait_details(
WaitDetails::builder()
.scheduled_end_timestamp(aws_smithy_types::DateTime::from(end))
.build(),
)
.build()
.expect("all required fields set"),
);
}
if req.empty_number == 1
&& let Some(start) = ever(req, name, &OperationAction::Start)
{
ops.push(wait_record(req, start, OperationStatus::Succeeded, None));
}
}
Ok(ops)
}));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let w1 = ctx.wait(secs(1)).name("w1").spawn();
let w2 = ctx.wait(secs(1)).name("w2").spawn();
let s = spawned_sleeping_step(&ctx, "s", secs(3));
let (a, b) = tokio::join!(w1, w2);
a.map_err(wire_err)?;
b.map_err(wire_err)?;
s.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_eq!(client.empty_calls().len(), 1);
let updates = client.updates();
let mut expected: Vec<String> = ["w1", "w2"]
.iter()
.map(|n| {
named(&updates, n, &OperationAction::Start)
.expect("the wait started")
.id()
.to_owned()
})
.collect();
expected.sort();
assert_eq!(coordinator.wake_log(), expected);
}
#[tokio::test(start_paused = true)]
async fn lone_wait_starts_no_refresh() {
let client = Arc::new(RefreshClient::new(|_| Ok(Vec::new())));
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
ctx.wait(secs(1)).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Pending);
tokio::time::sleep(secs(60)).await;
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
}
#[tokio::test(start_paused = true)]
async fn refresh_in_flight_finishes_and_readies_first() {
let client = Arc::new(
RefreshClient::new(|req| {
let mut ops = Vec::new();
if let Some(start) = this(req, "w", &OperationAction::Start) {
ops.push(wait_record(
req,
start,
OperationStatus::Started,
Some(secs(1)),
));
}
if req.empty_number == 1
&& let Some(start) = ever(req, "w", &OperationAction::Start)
{
ops.push(wait_record(req, start, OperationStatus::Succeeded, None));
}
Ok(ops)
})
.with_empty_delay(secs(1)),
);
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let w = ctx.wait(secs(1)).name("w").spawn();
let s = spawned_sleeping_step(&ctx, "s", millis(1500));
w.await.map_err(wire_err)?;
s.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_times(&client.empty_answers(), &[secs(2)]);
}
#[tokio::test(start_paused = true)]
async fn no_refresh_within_one_second_of_deadline() {
let client = Arc::new(callback_client(true));
let outcome = callback_beside_step(&client, Some(millis(1500)), secs(3)).await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
}
#[tokio::test(start_paused = true)]
async fn refresh_stops_under_one_second_left() {
let client = Arc::new(callback_client(false));
let outcome = callback_beside_step(&client, Some(millis(2500)), secs(3)).await;
assert_eq!(outcome, InvocationOutcome::Pending);
assert_times(&client.empty_calls(), &[secs(1)]);
}
#[tokio::test(start_paused = true)]
async fn refresh_failure_keeps_classification() {
let client = Arc::new(RefreshClient::new(|req| {
if req.empty_number > 0 {
return Err(ExecutionClientError::from_retryable(
"throttled refresh".to_owned(),
));
}
Ok(callback_starts(req.updates))
}));
let outcome = callback_beside_step(&client, None, secs(3)).await;
let InvocationOutcome::Fault { message, .. } = &outcome else {
panic!("expected a fault, got {outcome:?}");
};
assert_eq!(
message,
"[Unrecoverable Invocation] Checkpoint failed: throttled refresh"
);
assert_eq!(client.empty_calls().len(), 1, "no refresh after a failure");
let rejected = ExecutionClientError::new_non_retryable("rejected refresh");
let expected = crate::error::checkpoint_execution_failure_wire(&rejected);
let client = Arc::new(RefreshClient::new(move |req| {
if req.empty_number > 0 {
return Err(rejected.clone());
}
Ok(callback_starts(req.updates))
}));
let outcome = callback_beside_step(&client, None, secs(3)).await;
let InvocationOutcome::Failed { error } = &outcome else {
panic!("expected a failed execution, got {outcome:?}");
};
assert_eq!(error.error_type(), expected.error_type());
assert_eq!(client.empty_calls().len(), 1, "no refresh after a failure");
}
async fn callback_beside_activity(ctx: crate::DurableContext) -> InvocationOutcome {
drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(secs(3))).await;
h.await.map_err(wire_err)
})
.await
.expect("no panic")
}
#[tokio::test(start_paused = true)]
async fn batched_refresh_failure_keeps_classification() {
let batched_ctx = |client: &Arc<RefreshClient>| {
crate::DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::ZERO),
)
};
let client = Arc::new(RefreshClient::new(|req| {
if req.empty_number > 0 {
return Err(ExecutionClientError::from_retryable(
"throttled refresh".to_owned(),
));
}
Ok(callback_starts(req.updates))
}));
let outcome = callback_beside_activity(batched_ctx(&client)).await;
let InvocationOutcome::Fault { message, .. } = &outcome else {
panic!("expected a fault, got {outcome:?}");
};
assert_eq!(
message,
"[Unrecoverable Invocation] Checkpoint failed: throttled refresh"
);
let rejected = ExecutionClientError::new_non_retryable("rejected refresh");
let expected = crate::error::checkpoint_execution_failure_wire(&rejected);
let client = Arc::new(RefreshClient::new(move |req| {
if req.empty_number > 0 {
return Err(rejected.clone());
}
Ok(callback_starts(req.updates))
}));
let outcome = callback_beside_activity(batched_ctx(&client)).await;
let InvocationOutcome::Failed { error } = &outcome else {
panic!("expected a failed execution, got {outcome:?}");
};
assert_eq!(error.error_type(), expected.error_type());
}
fn panic_message(payload: &(dyn std::any::Any + Send)) -> Option<&str> {
payload
.downcast_ref::<String>()
.map(String::as_str)
.or_else(|| payload.downcast_ref::<&str>().copied())
}
#[tokio::test(start_paused = true)]
async fn batched_client_panic_fails_invocation() {
let client = Arc::new(RefreshClient::new(|req| {
assert!(req.empty_number == 0, "batched refresh client panic");
Ok(callback_starts(req.updates))
}));
let ctx = crate::DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::ZERO),
);
let coordinator = Arc::clone(ctx.coordinator());
let ended = tokio::time::timeout(
secs(600),
drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(secs(3))).await;
h.await.map_err(wire_err)
}),
)
.await
.expect("the drive ends");
let Err(payload) = ended else {
panic!("expected the client panic, got {ended:?}");
};
assert_eq!(
panic_message(payload.as_ref()),
Some("batched refresh client panic")
);
assert_eq!(coordinator.tasks().live(), 0);
assert_eq!(coordinator.live_waits(), 0);
assert_times(&client.empty_calls(), &[secs(1)]);
}
#[tokio::test(start_paused = true)]
async fn kept_panic_ends_the_drive() {
let client = Arc::new(callback_client(false));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let ended = tokio::time::timeout(
secs(600),
drive_handler(ctx, None, |ctx| async move {
assert!(
ctx.coordinator()
.record_panic(Box::new("detached boom"))
.is_none()
);
ctx.hold_activity(std::future::pending::<()>()).await;
Ok("unreachable".to_owned())
}),
)
.await
.expect("the drive ends");
let Err(payload) = ended else {
panic!("expected the kept panic, got {ended:?}");
};
assert_eq!(panic_message(payload.as_ref()), Some("detached boom"));
assert_eq!(coordinator.tasks().live(), 0);
}
#[tokio::test(start_paused = true)]
async fn fatal_takes_precedence_over_kept_panic() {
struct PanicsOnDrop;
impl Drop for PanicsOnDrop {
#[expect(clippy::panic)]
fn drop(&mut self) {
panic!("payload drop");
}
}
let client = Arc::new(callback_client(false));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let ended = drive_handler(ctx, None, |ctx| async move {
ctx.coordinator()
.record_invocation_fault("FatalFirst", "fatal first".to_owned());
assert!(
ctx.coordinator()
.record_panic(Box::new(PanicsOnDrop))
.is_none()
);
Ok("done".to_owned())
})
.await;
let Ok(InvocationOutcome::Fault { message, .. }) = &ended else {
panic!("expected the fatal outcome, got {ended:?}");
};
assert!(message.contains("fatal first"), "{message}");
assert!(!coordinator.has_panic());
}
#[tokio::test(start_paused = true)]
async fn fatal_takes_precedence_over_nested_panicking_kept_payload() {
let client = Arc::new(callback_client(false));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let ended = drive_handler(ctx, None, |ctx| async move {
ctx.coordinator()
.record_invocation_fault("FatalFirst", "fatal first".to_owned());
assert!(
ctx.coordinator()
.record_panic(Box::new(super::task_gateway::NestedPanicPayload))
.is_none()
);
Ok("done".to_owned())
})
.await;
let Ok(InvocationOutcome::Fault { message, .. }) = &ended else {
panic!("expected the fatal outcome, got {ended:?}");
};
assert!(message.contains("fatal first"), "{message}");
assert!(!coordinator.has_panic());
}
#[tokio::test(start_paused = true)]
async fn refresh_failure_in_flight_when_handler_returns_keeps_classification() {
async fn run(failure: ExecutionClientError) -> (InvocationOutcome, Arc<RefreshClient>) {
let client = Arc::new(
RefreshClient::new(move |req| {
if req.empty_number > 0 {
return Err(failure.clone());
}
Ok(callback_starts(req.updates))
})
.with_empty_delay(millis(500)),
);
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(millis(1200))).await;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(coordinator.tasks().live(), 0);
(outcome, client)
}
let (outcome, client) = run(ExecutionClientError::from_retryable(
"throttled refresh".to_owned(),
))
.await;
let InvocationOutcome::Fault { message, .. } = &outcome else {
panic!("expected a fault, got {outcome:?}");
};
assert!(message.contains("throttled refresh"), "got {message}");
assert_times(&client.empty_calls(), &[secs(1)]);
let rejected = ExecutionClientError::new_non_retryable("rejected refresh");
let expected = crate::error::checkpoint_execution_failure_wire(&rejected);
let (outcome, client) = run(rejected).await;
let InvocationOutcome::Failed { error } = &outcome else {
panic!("expected a failed execution, got {outcome:?}");
};
assert_eq!(error.error_type(), expected.error_type());
assert_times(&client.empty_calls(), &[secs(1)]);
}
struct RefreshPanic(u32);
async fn drive_refresh_panic_after_return(
fatal: bool,
) -> (
Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>,
Arc<crate::coordinator::InvocationCoordinator>,
) {
let client = Arc::new(
RefreshClient::new(|req| {
if req.empty_number > 0 {
std::panic::panic_any(RefreshPanic(7));
}
Ok(callback_starts(req.updates))
})
.with_empty_delay(millis(500)),
);
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, move |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(millis(1200))).await;
if fatal {
ctx.coordinator()
.record_invocation_fault("FatalFirst", "fatal first".to_owned());
}
Ok("done".to_owned())
})
.await;
(outcome, coordinator)
}
#[tokio::test(start_paused = true)]
async fn refresh_panic_after_handler_returns() {
let (outcome, coordinator) = drive_refresh_panic_after_return(false).await;
let Err(payload) = outcome else {
panic!("expected the refresh panic, got {outcome:?}");
};
assert_eq!(payload.downcast_ref::<RefreshPanic>().map(|p| p.0), Some(7));
assert_eq!(coordinator.tasks().live(), 0);
}
#[tokio::test(start_paused = true)]
async fn fatal_wins_over_refresh_panic_after_handler_returns() {
let (outcome, coordinator) = drive_refresh_panic_after_return(true).await;
let Ok(InvocationOutcome::Fault { message, .. }) = &outcome else {
panic!("expected the fatal outcome, got {outcome:?}");
};
assert!(message.contains("fatal first"), "{message}");
assert!(!coordinator.has_panic());
assert_eq!(coordinator.tasks().live(), 0);
}
#[tokio::test(start_paused = true)]
async fn batched_refresh_failure_when_handler_panics_keeps_classification() {
async fn run(failure: ExecutionClientError) -> InvocationOutcome {
let client = Arc::new(
RefreshClient::new(move |req| {
if req.empty_number > 0 {
return Err(failure.clone());
}
Ok(callback_starts(req.updates))
})
.with_empty_delay(millis(500)),
);
let ctx = crate::DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(crate::engine::CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::ZERO),
);
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(millis(1200))).await;
panic!("handler panic beside a refresh");
})
.await
.expect("the refresh failure wins over the handler panic");
assert_eq!(coordinator.tasks().live(), 0);
assert_times(&client.empty_calls(), &[secs(1)]);
outcome
}
let outcome = run(ExecutionClientError::from_retryable(
"throttled refresh".to_owned(),
))
.await;
let InvocationOutcome::Fault { message, .. } = &outcome else {
panic!("expected a fault, got {outcome:?}");
};
assert!(message.contains("throttled refresh"), "got {message}");
let rejected = ExecutionClientError::new_non_retryable("rejected refresh");
let expected = crate::error::checkpoint_execution_failure_wire(&rejected);
let outcome = run(rejected).await;
let InvocationOutcome::Failed { error } = &outcome else {
panic!("expected a failed execution, got {outcome:?}");
};
assert_eq!(error.error_type(), expected.error_type());
}
#[tokio::test(start_paused = true)]
async fn refresh_client_panic_cleans_up_and_propagates() {
let client = Arc::new(RefreshClient::new(|req| {
assert!(req.empty_number == 0, "refresh client panic");
Ok(callback_starts(req.updates))
}));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let ended = drive_handler(ctx, None, |ctx| async move {
let step = spawned_sleeping_step(&ctx, "s", secs(3));
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
step.await.map_err(wire_err)?;
h.await.map_err(wire_err)
})
.await;
let Err(payload) = ended else {
panic!("expected the client panic, got {ended:?}");
};
let message = payload
.downcast_ref::<String>()
.map(String::as_str)
.or_else(|| payload.downcast_ref::<&str>().copied());
assert_eq!(message, Some("refresh client panic"));
assert_eq!(coordinator.tasks().live(), 0);
assert_eq!(coordinator.live_waits(), 0);
assert_times(&client.empty_calls(), &[secs(1)]);
}
#[tokio::test(start_paused = true)]
async fn no_refresh_after_invocation_ends() {
let client = Arc::new(callback_client(false));
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
sleeping_step(&ctx, "s", millis(2500))
.await
.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
tokio::time::sleep(secs(120)).await;
assert_times(&client.empty_calls(), &[secs(1), secs(2)]);
}
#[tokio::test(start_paused = true)]
async fn refresh_in_flight_when_handler_returns_completes() {
let client = Arc::new(
RefreshClient::new(|req| {
let mut ops = callback_starts(req.updates);
if req.empty_number > 0
&& req.now >= millis(1400)
&& let Some(start) = ever(req, "cb", &OperationAction::Start)
{
ops.push(callback_record(
start,
OperationStatus::Succeeded,
Some("\"ok\""),
));
}
Ok(ops)
})
.with_empty_delay(millis(500)),
);
let ctx = refresh_ctx(&client);
let log_ctx = ctx.clone();
let coordinator = Arc::clone(ctx.coordinator());
let outcome = drive_handler(ctx, None, |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
ctx.hold_activity(tokio::time::sleep(millis(1200))).await;
Ok("done".to_owned())
})
.await
.expect("no panic");
let returned = client.elapsed();
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_times(&client.empty_answers(), &[millis(1500)]);
assert!(returned >= millis(1500), "returned at {returned:?}");
let cb_id = named(&client.updates(), "cb", &OperationAction::Start)
.expect("the callback started")
.id()
.to_owned();
assert_eq!(
log_ctx.refresh_view(&cb_id).map(|(status, _)| status),
Some(CheckpointStatus::Succeeded)
);
assert_eq!(coordinator.tasks().live(), 0);
}
#[tokio::test(start_paused = true)]
async fn new_earlier_registration_rearms_scheduler() {
let client = Arc::new(RefreshClient::new(|req| {
let mut ops = callback_starts(req.updates);
if let Some(start) = this(req, "w", &OperationAction::Start) {
ops.push(wait_record(
req,
start,
OperationStatus::Started,
Some(millis(300)),
));
}
if req.empty_number == 1
&& let Some(start) = ever(req, "w", &OperationAction::Start)
{
ops.push(wait_record(req, start, OperationStatus::Succeeded, None));
}
Ok(ops)
}));
let ctx = refresh_ctx(&client);
let waited = Arc::new(Mutex::new(None));
let handler_waited = Arc::clone(&waited);
let handler_client = Arc::clone(&client);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let s = spawned_sleeping_step(&ctx, "s", secs(10));
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut h = std::pin::pin!(cb.spawn_result());
poll_once(h.as_mut()).await;
tokio::time::sleep(millis(200)).await;
let result = ctx.wait(secs(1)).name("w").await;
*handler_waited.lock().expect("lock") =
Some((result.is_ok(), handler_client.elapsed()));
s.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
let first = *client.empty_calls().first().expect("a refresh was sent");
assert_near(first, millis(500));
let (ok, at) = waited.lock().expect("lock").expect("the wait returned");
assert!(ok, "the wait returns Ok(())");
assert_near(at, millis(500));
}
#[tokio::test(start_paused = true)]
async fn retired_earliest_registration_rearms_scheduler() {
let client = Arc::new(RefreshClient::new(|req| {
let mut ops = callback_starts(req.updates);
if let Some(start) = this(req, "w", &OperationAction::Start) {
ops.push(wait_record(
req,
start,
OperationStatus::Started,
Some(secs(3)),
));
}
if req.empty_number == 1
&& let Some(start) = ever(req, "w", &OperationAction::Start)
{
ops.push(wait_record(req, start, OperationStatus::Succeeded, None));
}
Ok(ops)
}));
let ctx = refresh_ctx(&client);
let outcome = drive_handler(ctx, None, |ctx| async move {
let s = spawned_sleeping_step(&ctx, "s", secs(10));
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
let mut cb_h = cb.result();
poll_once(std::pin::Pin::new(&mut cb_h)).await;
let w = ctx.wait(secs(3)).name("w").spawn();
let (waited, ()) = tokio::join!(w, async move {
tokio::time::sleep(millis(500)).await;
drop(cb_h);
});
waited.map_err(wire_err)?;
s.await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_times(&client.empty_calls(), &[secs(3)]);
}
#[derive(Debug, Clone, Copy)]
enum Spawned {
Wait,
Callback,
Invoke,
Condition,
}
impl Spawned {
fn name(self) -> &'static str {
match self {
Self::Wait => "w",
Self::Callback => "cb",
Self::Invoke => "inv",
Self::Condition => "cond",
}
}
fn client(self) -> RefreshClient {
RefreshClient::new(move |req| {
let mut ops = callback_starts(req.updates);
if let Some(start) = this(req, "w", &OperationAction::Start) {
ops.push(wait_record(
req,
start,
OperationStatus::Started,
Some(secs(1)),
));
}
if let Some(retry) = this(req, "cond", &OperationAction::Retry) {
ops.push(condition_record(
req,
retry,
OperationStatus::Pending,
retry.payload(),
Some(secs(1)),
));
}
if req.empty_number == 1 {
let ready = match self {
Self::Wait => ever(req, "w", &OperationAction::Start)
.map(|s| wait_record(req, s, OperationStatus::Succeeded, None)),
Self::Callback => ever(req, "cb", &OperationAction::Start).map(|s| {
callback_record(s, OperationStatus::Succeeded, Some("\"ok\""))
}),
Self::Invoke => ever(req, "inv", &OperationAction::Start)
.map(|s| invoke_record(s, OperationStatus::Succeeded, Some("7"))),
Self::Condition => ever(req, "cond", &OperationAction::Retry).map(|r| {
condition_record(req, r, OperationStatus::Ready, r.payload(), None)
}),
};
ops.extend(ready);
}
Ok(ops)
})
}
async fn start(
self,
ctx: &crate::DurableContext,
checks: &Arc<Mutex<Vec<Duration>>>,
) -> Result<crate::DurableFuture<String>, crate::error::WireError> {
Ok(match self {
Self::Wait => {
let h = ctx.wait(secs(1)).name("w").spawn();
crate::DurableFuture::from_async(
async move { h.await.map(|()| "()".to_owned()) },
)
}
Self::Callback => {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
cb.spawn_result()
}
Self::Invoke => {
let h = ctx.invoke::<u32>("target", 1_u32).name("inv").spawn();
crate::DurableFuture::from_async(async move { h.await.map(|v| v.to_string()) })
}
Self::Condition => {
let h = condition(ctx, tokio::time::Instant::now(), checks).spawn();
crate::DurableFuture::from_async(async move { h.await.map(|v| v.to_string()) })
}
})
}
fn value(self) -> &'static str {
match self {
Self::Wait => "()",
Self::Callback => "ok",
Self::Invoke => "7",
Self::Condition => "2",
}
}
}
#[tokio::test(start_paused = true)]
async fn unawaited_spawned_operation_is_not_refreshed() {
for kind in [Spawned::Wait, Spawned::Callback, Spawned::Invoke] {
let client = Arc::new(kind.client());
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let seen = Arc::new(Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let handler_client = Arc::clone(&client);
let checks = Arc::new(Mutex::new(Vec::new()));
let outcome = drive_handler(ctx, None, move |ctx| async move {
let _h = kind.start(&ctx, &checks).await?;
let live = ctx
.hold_activity(async {
for _ in 0..1000 {
if handler_client.received(kind.name(), &OperationAction::Start)
&& ctx.coordinator().live_waits() == 1
{
return Some(ctx.coordinator().live_waits());
}
tokio::task::yield_now().await;
}
None
})
.await;
*handler_seen.lock().expect("lock") = Some(live);
sleeping_step(&ctx, "s", secs(3)).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
let live = seen.lock().expect("lock").expect("the handler ran");
assert_eq!(live, Some(1), "{kind:?}: the operation is a live wait");
assert_eq!(
client.empty_calls(),
Vec::<Duration>::new(),
"{kind:?}: no refresh"
);
assert_eq!(
outcome,
InvocationOutcome::Complete("done".to_owned()),
"{kind:?}"
);
assert_eq!(coordinator.live_waits(), 0, "{kind:?}");
}
}
#[tokio::test(start_paused = true)]
async fn unawaited_spawned_condition_is_refreshed() {
let client = Arc::new(Spawned::Condition.client());
let ctx = refresh_ctx(&client);
let checks = Arc::new(Mutex::new(Vec::new()));
let handler_checks = Arc::clone(&checks);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let _h = Spawned::Condition.start(&ctx, &handler_checks).await?;
sleeping_step(&ctx, "s", secs(3)).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
assert_times(&checks.lock().expect("lock"), &[secs(0), secs(1)]);
assert!(
!client.empty_calls().is_empty(),
"the condition is refreshed"
);
}
#[tokio::test(start_paused = true)]
async fn dropped_callback_spawn_result_keeps_waiting() {
let client = Arc::new(Spawned::Callback.client());
let ctx = refresh_ctx(&client);
let seen = Arc::new(Mutex::new(None));
let handler_seen = Arc::clone(&seen);
let outcome = drive_handler(ctx, None, move |ctx| async move {
let cb = ctx
.create_callback::<String>()
.name("cb")
.await
.map_err(wire_err)?;
drop(cb.spawn_result());
let live = ctx
.hold_activity(async {
for _ in 0..1000 {
if ctx.coordinator().live_waits() == 1 {
break;
}
tokio::task::yield_now().await;
}
ctx.coordinator().live_waits()
})
.await;
*handler_seen.lock().expect("lock") = Some(live);
sleeping_step(&ctx, "s", secs(1)).await.map_err(wire_err)?;
Ok("done".to_owned())
})
.await
.expect("no panic");
assert_eq!(*seen.lock().expect("lock"), Some(1), "the wait stays live");
assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
}
#[tokio::test(start_paused = true)]
async fn spawned_operation_refresh_starts_at_first_poll() {
for kind in [Spawned::Wait, Spawned::Callback, Spawned::Invoke] {
let client = Arc::new(kind.client());
let ctx = refresh_ctx(&client);
let resolved = Arc::new(Mutex::new(None));
let handler_resolved = Arc::clone(&resolved);
let handler_client = Arc::clone(&client);
let checks = Arc::new(Mutex::new(Vec::new()));
let outcome = drive_handler(ctx, None, move |ctx| async move {
let h = kind.start(&ctx, &checks).await?;
let s = spawned_sleeping_step(&ctx, "s", secs(4));
tokio::time::sleep(secs(2)).await;
let value = h.await.map_err(wire_err)?;
*handler_resolved.lock().expect("lock") = Some(handler_client.elapsed());
s.await.map_err(wire_err)?;
Ok(value)
})
.await
.expect("no panic");
let expected_at = match kind {
Spawned::Wait | Spawned::Condition => secs(2),
Spawned::Callback | Spawned::Invoke => secs(3),
};
assert_eq!(
outcome,
InvocationOutcome::Complete(kind.value().to_owned()),
"{kind:?}"
);
let first = *client.empty_calls().first().expect("a refresh was sent");
assert_near(first, expected_at);
let at = resolved.lock().expect("lock").expect("the handle resolved");
assert_near(at, expected_at);
}
}
#[tokio::test(start_paused = true)]
async fn refused_scheduler_permit_sends_nothing() {
let client = Arc::new(RefreshClient::new(|_| Ok(Vec::new())));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
coordinator.record_fatal("Fatal".to_owned(), "stop".to_owned());
let _wait = coordinator
.register_test_wait("op", std::task::Waker::noop())
.expect("running");
let mut scheduler = std::pin::pin!(super::refresh_scheduler(ctx));
let polled = tokio::time::timeout(secs(30), scheduler.as_mut()).await;
assert!(polled.is_err(), "the scheduler stays pending");
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
assert_eq!(coordinator.permits(PermitKind::Scheduler), 0);
}
#[tokio::test(start_paused = true)]
async fn refused_scheduler_permit_while_running_keeps_refreshing() {
let client = Arc::new(RefreshClient::new(|_| Ok(Vec::new())));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let w1 = coordinator
.register_test_wait("w1", std::task::Waker::noop())
.expect("running");
let mut scheduler = std::pin::pin!(super::refresh_scheduler(ctx));
let polled = tokio::time::timeout(secs(2), scheduler.as_mut()).await;
assert!(polled.is_err(), "the scheduler stays pending");
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
drop(w1);
let _activity = coordinator.acquire(PermitKind::Activity).expect("running");
let _w2 = coordinator
.register_test_wait("w2", std::task::Waker::noop())
.expect("running");
let polled = tokio::time::timeout(millis(1500), scheduler.as_mut()).await;
assert!(polled.is_err(), "the scheduler stays pending");
assert_eq!(client.empty_calls(), vec![secs(3)]);
}
#[tokio::test(start_paused = true)]
async fn refused_scheduler_permit_after_withdrawal_or_finish_sends_nothing() {
for withdraw in [true, false] {
let client = Arc::new(RefreshClient::new(|_| Ok(Vec::new())));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
let _wait = coordinator
.register_test_wait("op", std::task::Waker::noop())
.expect("running");
if withdraw {
coordinator.withdraw("arn", "request", || {});
} else {
coordinator.finish();
}
let mut scheduler = std::pin::pin!(super::refresh_scheduler(ctx));
let polled = tokio::time::timeout(secs(30), scheduler.as_mut()).await;
assert!(polled.is_err(), "the scheduler stays pending");
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
assert_eq!(coordinator.permits(PermitKind::Scheduler), 0);
}
}
#[tokio::test(start_paused = true)]
async fn refused_scheduler_permit_after_panic_sends_nothing() {
let client = Arc::new(RefreshClient::new(|_| Ok(Vec::new())));
let ctx = refresh_ctx(&client);
let coordinator = Arc::clone(ctx.coordinator());
assert!(coordinator.record_panic(Box::new("boom")).is_none());
let _wait = coordinator
.register_test_wait("op", std::task::Waker::noop())
.expect("running");
let mut scheduler = std::pin::pin!(super::refresh_scheduler(ctx));
let polled = tokio::time::timeout(secs(30), scheduler.as_mut()).await;
assert!(polled.is_err(), "the scheduler stays pending");
assert_eq!(client.empty_calls(), Vec::<Duration>::new());
assert_eq!(coordinator.permits(PermitKind::Scheduler), 0);
}
#[cfg(feature = "test-util")]
#[tokio::test(start_paused = true)]
async fn replay_after_suspension_matches_in_process() {
let client = Arc::new(test3_client());
let ctx = refresh_ctx(&client);
let log_ctx = ctx.clone();
let outcome = drive_handler(ctx, None, |ctx| async move {
parallel_wait_then_step(ctx)
.await
.map_err(|e| crate::error::WireError::new(None::<String>, Some(e.to_string())))
})
.await
.expect("no panic");
let InvocationOutcome::Complete(in_process) = outcome else {
panic!("expected Complete, got {outcome:?}");
};
let mut seen: Vec<String> = Vec::new();
let mut in_process_ops = Vec::new();
for update in client.updates() {
if seen.contains(&update.id().to_owned()) {
continue;
}
seen.push(update.id().to_owned());
let (status, _) = log_ctx.refresh_view(update.id()).expect("recorded");
in_process_ops.push((
update.name().map(str::to_owned),
update.r#type().as_str().to_owned(),
status,
));
}
let result = crate::test_util::LocalRunner::new()
.run(
|(): (), ctx: crate::DurableContext| parallel_wait_then_step(ctx),
(),
)
.await
.expect("the run finishes");
assert!(result.invocation_count() > 1, "LocalRunner suspended");
assert_eq!(
result.output().map(String::as_str),
Some(in_process.as_str())
);
let replayed_ops: Vec<_> = result
.operations()
.iter()
.map(|op| {
let status = match op.status() {
crate::OperationStatus::Succeeded => CheckpointStatus::Succeeded,
crate::OperationStatus::Failed => CheckpointStatus::Failed,
other => CheckpointStatus::Unknown(format!("{other:?}")),
};
(
op.name().map(str::to_owned),
op.kind().wire_str().to_owned(),
status,
)
})
.collect();
assert_eq!(replayed_ops, in_process_ops);
}
}
#[cfg(test)]
#[expect(clippy::panic)] mod panic_owner {
use super::{InvocationOutcome, drive_outcome, drive_with_scheduler};
use crate::client::{
CheckpointOutput, ExecutionClient, ExecutionClientError, GetStateOutput,
InMemoryExecutionClient,
};
use crate::context::DurableContext;
use crate::coordinator::InvocationCoordinator;
use crate::engine::CheckpointLog;
use crate::future::DurableFuture;
use crate::task::TrackedJoinSet;
use aws_sdk_lambda::types::OperationUpdate;
use std::any::Any;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
type Payload = Box<dyn Any + Send + 'static>;
type Driven = Result<InvocationOutcome, Payload>;
fn ctx_with(client: Arc<dyn ExecutionClient>) -> DurableContext {
DurableContext::new_root_with_client(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
client,
"token0".to_owned(),
)
}
fn live_ctx() -> DurableContext {
ctx_with(Arc::new(InMemoryExecutionClient::new(Vec::new())))
}
struct Tagged(u32, Arc<AtomicUsize>);
impl Drop for Tagged {
fn drop(&mut self) {
self.1.fetch_add(1, Ordering::SeqCst);
}
}
fn panic_tagged(n: u32, drops: &Arc<AtomicUsize>) -> ! {
std::panic::panic_any(Tagged(n, Arc::clone(drops)))
}
fn counter() -> Arc<AtomicUsize> {
Arc::new(AtomicUsize::new(0))
}
fn tag(payload: &(dyn Any + Send)) -> Option<u32> {
payload.downcast_ref::<Tagged>().map(|p| p.0)
}
fn returned_tag(driven: &Driven) -> Option<u32> {
let Err(payload) = driven else {
panic!("expected a panic, got {driven:?}");
};
tag(payload.as_ref())
}
struct PanicsOnDrop(u32, Arc<AtomicUsize>);
impl Drop for PanicsOnDrop {
fn drop(&mut self) {
panic_tagged(self.0, &self.1);
}
}
struct DropFlag(Arc<AtomicBool>);
impl Drop for DropFlag {
fn drop(&mut self) {
self.0.store(true, Ordering::SeqCst);
}
}
async fn until(ctx: &DurableContext, done: impl Fn() -> bool) {
ctx.hold_activity(async {
for _ in 0..1000 {
if done() {
return;
}
tokio::task::yield_now().await;
}
panic!("the condition did not hold within 1,000 yields");
})
.await;
}
async fn drive<F, Fut>(ctx: DurableContext, handler: F) -> (Driven, usize)
where
F: FnOnce(DurableContext) -> Fut,
Fut: Future<Output = Result<String, crate::error::WireError>> + Send + 'static,
{
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let tasks = Arc::clone(coordinator.tasks());
let root = ctx.clone();
let driven = drive_outcome(handler(ctx), ownership, coordinator, root, None).await;
(driven, tasks.live())
}
fn spawn_holding<G: Send + 'static>(
ctx: &DurableContext,
name: &str,
guard: G,
started: &Arc<AtomicBool>,
) -> DurableFuture<()> {
let guard = Arc::new(Mutex::new(Some(guard)));
let started = Arc::clone(started);
ctx.step(move |_| {
let guard = guard.lock().expect("lock").take();
let started = Arc::clone(&started);
async move {
let _guard = guard;
started.store(true, Ordering::SeqCst);
std::future::pending::<()>().await;
Ok(())
}
})
.name(name)
.spawn()
}
fn spawn_panicking(
ctx: &DurableContext,
n: u32,
drops: &Arc<AtomicUsize>,
) -> DurableFuture<()> {
let drops = Arc::clone(drops);
ctx.step(move |_| {
let drops = Arc::clone(&drops);
async move { panic_tagged(n, &drops) }
})
.name("boom")
.spawn()
}
#[tokio::test]
async fn dropped_handle_of_panicked_task_fails_invocation() {
let (driven, live) = drive(live_ctx(), |ctx| async move {
let step = spawn_panicking(&ctx, 6, &counter());
let tasks = Arc::clone(ctx.coordinator().tasks());
until(&ctx, || tasks.live() == 0).await;
drop(step);
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(6));
assert_eq!(live, 0);
}
#[tokio::test]
async fn escaped_handle_with_unread_panic_fails_invocation() {
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let kept = Arc::clone(&escaped);
let (driven, live) = drive(live_ctx(), |ctx| async move {
let step = spawn_panicking(&ctx, 7, &counter());
let tasks = Arc::clone(ctx.coordinator().tasks());
until(&ctx, || tasks.live() == 0).await;
*kept.lock().expect("lock") = Some(step);
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(7));
assert!(
escaped.lock().expect("lock").is_some(),
"the test still holds the handle"
);
assert_eq!(live, 0);
}
#[tokio::test]
async fn panic_while_aborting_dropped_task_fails_invocation() {
let (driven, live) = drive(live_ctx(), |ctx| async move {
let started = Arc::new(AtomicBool::new(false));
let step = spawn_holding(&ctx, "hold", PanicsOnDrop(8, counter()), &started);
until(&ctx, || started.load(Ordering::SeqCst)).await;
drop(step);
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(8));
assert_eq!(live, 0);
}
#[tokio::test]
async fn queued_join_set_panic_fails_invocation() {
let (driven, live) = drive(live_ctx(), |ctx| async move {
let drops = counter();
let mut set = TrackedJoinSet::<()>::new(Arc::clone(ctx.coordinator()));
set.spawn(
crate::task::TaskContext::none(),
ctx.task_ownership(),
async move { panic_tagged(9, &drops) },
)
.expect("runtime");
let tasks = Arc::clone(ctx.coordinator().tasks());
until(&ctx, || tasks.live() == 0).await;
drop(set);
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(9));
assert_eq!(live, 0);
}
struct PendingThenDropPanics {
started: Arc<AtomicBool>,
_guard: PanicsOnDrop,
}
impl Future for PendingThenDropPanics {
type Output = ();
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
self.started.store(true, Ordering::SeqCst);
Poll::Pending
}
}
#[tokio::test]
async fn join_set_panic_after_drop_fails_invocation() {
let (driven, live) = drive(live_ctx(), |ctx| async move {
let started = Arc::new(AtomicBool::new(false));
let mut set = TrackedJoinSet::<()>::new(Arc::clone(ctx.coordinator()));
set.spawn(
crate::task::TaskContext::none(),
ctx.task_ownership(),
PendingThenDropPanics {
started: Arc::clone(&started),
_guard: PanicsOnDrop(10, counter()),
},
)
.expect("runtime");
until(&ctx, || started.load(Ordering::SeqCst)).await;
drop(set);
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(10));
assert_eq!(live, 0);
}
struct FatalOnDrop(Arc<InvocationCoordinator>);
impl Drop for FatalOnDrop {
fn drop(&mut self) {
self.0
.record_fatal("Fatal".to_owned(), "during cleanup".to_owned());
}
}
#[tokio::test(flavor = "current_thread")]
async fn fatal_after_panic_wins() {
let drops = counter();
let handler_drops = Arc::clone(&drops);
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let kept = Arc::clone(&escaped);
let (driven, live) = drive(live_ctx(), |ctx| async move {
let started = Arc::new(AtomicBool::new(false));
let guard = FatalOnDrop(Arc::clone(ctx.coordinator()));
*kept.lock().expect("lock") = Some(spawn_holding(&ctx, "hold", guard, &started));
until(&ctx, || started.load(Ordering::SeqCst)).await;
assert!(ctx.coordinator().fatal_error().is_none(), "no fatal yet");
panic_tagged(4, &handler_drops)
})
.await;
let Ok(InvocationOutcome::Failed { error }) = &driven else {
panic!("expected the fatal outcome, got {driven:?}");
};
assert_eq!(error.error_message(), Some("during cleanup"));
assert_eq!(drops.load(Ordering::SeqCst), 1, "the payload was dropped");
assert_eq!(live, 0);
}
#[tokio::test]
async fn fatal_before_panic_wins() {
let drops = counter();
let handler_drops = Arc::clone(&drops);
let (driven, live) = drive(live_ctx(), |ctx| async move {
ctx.coordinator()
.record_fatal("Fatal".to_owned(), "first".to_owned());
panic_tagged(5, &handler_drops)
})
.await;
let Ok(InvocationOutcome::Failed { error }) = &driven else {
panic!("expected the fatal outcome, got {driven:?}");
};
assert_eq!(error.error_message(), Some("first"));
assert_eq!(drops.load(Ordering::SeqCst), 1, "the payload was dropped");
assert_eq!(live, 0);
}
#[tokio::test(flavor = "current_thread")]
async fn first_recorded_panic_wins() {
let a_drops = counter();
let b_drops = counter();
let (handler_a, handler_b) = (Arc::clone(&a_drops), Arc::clone(&b_drops));
let escaped: Arc<Mutex<Option<DurableFuture<()>>>> = Arc::new(Mutex::new(None));
let kept = Arc::clone(&escaped);
let (driven, live) = drive(live_ctx(), |ctx| async move {
let started = Arc::new(AtomicBool::new(false));
let guard = PanicsOnDrop(182, handler_b);
*kept.lock().expect("lock") = Some(spawn_holding(&ctx, "hold", guard, &started));
until(&ctx, || started.load(Ordering::SeqCst)).await;
panic_tagged(181, &handler_a)
})
.await;
assert_eq!(returned_tag(&driven), Some(181));
assert_eq!(b_drops.load(Ordering::SeqCst), 1, "B was dropped");
assert_eq!(
a_drops.load(Ordering::SeqCst),
0,
"A is returned, not dropped"
);
assert_eq!(live, 0);
}
#[tokio::test]
async fn cleanup_continues_after_a_panic() {
let flag = Arc::new(AtomicBool::new(false));
let handler_flag = Arc::clone(&flag);
let escaped: Arc<Mutex<Vec<DurableFuture<()>>>> = Arc::new(Mutex::new(Vec::new()));
let kept = Arc::clone(&escaped);
let (driven, live) = drive(live_ctx(), |ctx| async move {
let first = Arc::new(AtomicBool::new(false));
let second = Arc::new(AtomicBool::new(false));
let a = spawn_holding(&ctx, "panics", PanicsOnDrop(19, counter()), &first);
let b = spawn_holding(&ctx, "flag", DropFlag(handler_flag), &second);
kept.lock().expect("lock").extend([a, b]);
until(&ctx, || {
first.load(Ordering::SeqCst) && second.load(Ordering::SeqCst)
})
.await;
Ok("done".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(19));
assert!(flag.load(Ordering::SeqCst), "the second task was dropped");
assert_eq!(live, 0);
}
#[tokio::test]
async fn root_drop_panic_during_cleanup_fails_invocation() {
let (driven, live) = drive(live_ctx(), |ctx| async move {
let _guard = PanicsOnDrop(20, counter());
ctx.wait(Duration::from_secs(10))
.name("timer")
.await
.map_err(|err| {
crate::error::WireError::new(None::<String>, Some(err.to_string()))
})?;
Ok("unreachable".to_owned())
})
.await;
assert_eq!(returned_tag(&driven), Some(20));
assert_eq!(live, 0);
}
struct PanickingScheduler(u32);
impl Future for PanickingScheduler {
type Output = ();
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
panic_tagged(self.0, &counter())
}
}
struct DropPanickingScheduler {
_guard: PanicsOnDrop,
}
impl Future for DropPanickingScheduler {
type Output = ();
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
Poll::Pending
}
}
async fn drive_with<S: Future<Output = ()>>(
scheduler: S,
wait_forever: bool,
) -> (Driven, usize, bool) {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let tasks = Arc::clone(coordinator.tasks());
let dropped = Arc::new(AtomicBool::new(false));
let body_dropped = Arc::clone(&dropped);
let handler = async move {
let started = Arc::new(AtomicBool::new(false));
let _step = spawn_holding(&ctx, "hold", DropFlag(body_dropped), &started);
if wait_forever {
ctx.hold_activity(std::future::pending::<()>()).await;
}
until(&ctx, || started.load(Ordering::SeqCst)).await;
Ok("done".to_owned())
};
let driven = drive_with_scheduler(
crate::task::TaskContext::none(),
handler,
ownership,
coordinator,
scheduler,
None,
)
.await
.map(|(outcome, _)| outcome);
(driven, tasks.live(), dropped.load(Ordering::SeqCst))
}
#[tokio::test]
async fn scheduler_poll_or_drop_panic_fails_after_cleanup() {
let (driven, live, _) = drive_with(PanickingScheduler(211), true).await;
assert_eq!(returned_tag(&driven), Some(211));
assert_eq!(live, 0);
let (driven, live, dropped) = drive_with(
DropPanickingScheduler {
_guard: PanicsOnDrop(212, counter()),
},
false,
)
.await;
assert_eq!(returned_tag(&driven), Some(212));
assert!(dropped, "the cleanup dropped the step after the panic");
assert_eq!(live, 0);
}
struct SelfReferential(Option<DurableContext>, Arc<AtomicUsize>);
impl Drop for SelfReferential {
fn drop(&mut self) {
drop(self.0.take());
self.1.fetch_add(1, Ordering::SeqCst);
std::panic::panic_any(super::task_gateway::NestedPanicPayload);
}
}
async fn poll_into_cleanup(
drive: &mut Pin<Box<impl Future<Output = Driven>>>,
coordinator: &InvocationCoordinator,
handler_dropped: &AtomicBool,
) {
for _ in 0..1000 {
let polled = std::future::poll_fn(|cx| Poll::Ready(drive.as_mut().poll(cx))).await;
assert!(polled.is_pending(), "the drive waits for the detached task");
if handler_dropped.load(Ordering::SeqCst)
&& coordinator.phase() == crate::coordinator::Phase::Finished
{
return;
}
tokio::task::yield_now().await;
}
panic!("the drive did not reach its cleanup");
}
#[tokio::test]
async fn cancelled_drive_discards_self_referential_panic() {
for late in [false, true] {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
let root = ctx.clone();
let drops = counter();
let handler_drops = Arc::clone(&drops);
let handler_dropped = Arc::new(AtomicBool::new(false));
let flag = DropFlag(Arc::clone(&handler_dropped));
let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
let handler = async move {
let _flag = flag;
if late {
let payload_ctx = ctx.clone();
let detached = async move {
drop(release_rx.await);
std::panic::panic_any(SelfReferential(Some(payload_ctx), handler_drops));
};
crate::task::spawn_detached(ctx.coordinator(), detached).expect("runtime");
} else {
let detached = async move {
drop(release_rx.await);
};
crate::task::spawn_detached(ctx.coordinator(), detached).expect("runtime");
let payload = SelfReferential(Some(ctx.clone()), handler_drops);
assert!(ctx.coordinator().record_panic(Box::new(payload)).is_none());
}
drop(ctx);
if late {
return Ok("done".to_owned());
}
std::future::pending::<Result<String, crate::error::WireError>>().await
};
let mut drive = Box::pin(drive_outcome(
handler,
ownership,
Arc::clone(&coordinator),
root,
None,
));
poll_into_cleanup(&mut drive, &coordinator, &handler_dropped).await;
let dropped =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || drop(drive)));
assert!(dropped.is_ok(), "no panic escapes the drop");
release_tx.send(()).expect("the detached task waits");
coordinator.tasks().wait_idle().await;
assert_eq!(drops.load(Ordering::SeqCst), 1, "late = {late}");
assert!(!coordinator.has_panic(), "late = {late}");
assert!(coordinator.take_panic().is_none());
}
}
#[derive(Debug)]
struct PanickingClient {
n: u32,
on_drop: bool,
}
struct ReadyThenDropPanics(u32);
impl Future for ReadyThenDropPanics {
type Output = Result<CheckpointOutput, ExecutionClientError>;
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
Poll::Ready(Ok(CheckpointOutput {
checkpoint_token: Some("token1".to_owned()),
updated_operations: Vec::new(),
next_marker: None,
}))
}
}
impl Drop for ReadyThenDropPanics {
fn drop(&mut self) {
panic_tagged(self.0, &counter());
}
}
impl ExecutionClient for PanickingClient {
fn checkpoint(
&self,
_arn: &str,
_token: &str,
_updates: Vec<OperationUpdate>,
) -> Pin<Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>>
{
let n = self.n;
if self.on_drop {
return Box::pin(ReadyThenDropPanics(n));
}
Box::pin(async move { panic_tagged(n, &counter()) })
}
fn get_state(
&self,
_arn: &str,
_token: &str,
_marker: Option<&str>,
) -> Pin<Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>>
{
Box::pin(async {
Ok(GetStateOutput {
operations: Vec::new(),
next_marker: None,
})
})
}
}
#[tokio::test]
async fn detached_future_drop_panic_fails_invocation() {
let ctx = DurableContext::new_root_with_client_and_coalescer(
"arn:aws:lambda:us-east-1:1:function:test".to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::new(PanickingClient {
n: 15,
on_drop: true,
}) as Arc<dyn ExecutionClient>,
"token0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::ZERO),
);
let (driven, live) = tokio::time::timeout(
Duration::from_secs(10),
drive(ctx, |ctx| async move {
ctx.step(|_| async { Ok(1_u32) })
.name("s")
.await
.map_err(|err| {
crate::error::WireError::new(None::<String>, Some(err.to_string()))
})?;
Ok("done".to_owned())
}),
)
.await
.expect("the drive ends");
assert_eq!(returned_tag(&driven), Some(15));
assert_eq!(live, 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn write_waiter_stays_pending_when_client_panics() {
let ctx = ctx_with(Arc::new(PanickingClient {
n: 17,
on_drop: false,
}));
let second_poll: Arc<Mutex<Option<bool>>> = Arc::new(Mutex::new(None));
let observed = Arc::clone(&second_poll);
let (driven, live) = tokio::time::timeout(
Duration::from_secs(10),
drive(ctx, |ctx| async move {
let update = OperationUpdate::builder()
.id("w")
.r#type(aws_sdk_lambda::types::OperationType::Step)
.action(aws_sdk_lambda::types::OperationAction::Start)
.build()
.expect("valid update");
let mut write = std::pin::pin!(ctx.checkpoint_updates(vec![update]));
std::future::poll_fn(|cx| {
assert!(
write.as_mut().poll(cx).is_pending(),
"the write runs on its task"
);
tokio::task::block_in_place(|| {
let start = std::time::Instant::now();
while !ctx.coordinator().has_panic() {
assert!(
start.elapsed() < Duration::from_secs(5),
"the client panics"
);
std::thread::sleep(Duration::from_millis(1));
}
});
*observed.lock().expect("lock") = Some(write.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
std::future::pending::<Result<String, crate::error::WireError>>().await
}),
)
.await
.expect("the drive ends");
assert_eq!(
*second_poll.lock().expect("lock"),
Some(true),
"the waiter stays pending"
);
assert_eq!(returned_tag(&driven), Some(17));
assert_eq!(live, 0);
}
struct ProbeOnDrop(Arc<AtomicBool>);
impl Future for ProbeOnDrop {
type Output = ();
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
Poll::Pending
}
}
impl Drop for ProbeOnDrop {
fn drop(&mut self) {
let (tx, rx) = std::sync::mpsc::channel();
#[expect(clippy::disallowed_methods)]
let probe = tokio::runtime::Handle::current().spawn(async move {
tx.send(()).ok();
});
drop(probe);
if rx.recv_timeout(Duration::from_secs(10)).is_ok() {
self.0.store(true, Ordering::SeqCst);
}
}
}
#[test]
fn panic_recorded_before_the_drive_leaves_the_handler_unpolled() {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.expect("runtime");
let drops = counter();
let polled = Arc::new(AtomicBool::new(false));
let probe_ran = Arc::new(AtomicBool::new(false));
let handler_polled = Arc::clone(&polled);
let scheduler = ProbeOnDrop(Arc::clone(&probe_ran));
let driven = runtime.block_on(async {
let ctx = live_ctx();
let ownership = Arc::clone(ctx.task_ownership());
let coordinator = Arc::clone(ctx.coordinator());
assert!(
coordinator
.record_panic(Box::new(Tagged(119, Arc::clone(&drops))))
.is_none(),
"the coordinator keeps the panic"
);
let handler = async move {
handler_polled.store(true, Ordering::SeqCst);
std::future::pending::<Result<String, crate::error::WireError>>().await
};
drive_with_scheduler(
crate::task::TaskContext::none(),
handler,
ownership,
coordinator,
scheduler,
None,
)
.await
.map(|(outcome, _)| outcome)
});
assert!(probe_ran.load(Ordering::SeqCst), "the probe ran");
assert!(
!polled.load(Ordering::SeqCst),
"the handler is never polled"
);
assert_eq!(returned_tag(&driven), Some(119));
}
}
#[cfg(test)]
mod withdrawal_tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use aws_sdk_lambda::types::{
Operation, OperationAction, OperationStatus as WireStatus, OperationType, OperationUpdate,
};
use super::test_support::EventMessages;
use crate::client::{
CheckpointOutput, ExecutionClient, ExecutionClientError, ExecutionClientErrorScope,
GetStateOutput, InMemoryExecutionClient,
};
use crate::coordinator::{PermitKind, Phase, WITHDRAWAL_WARNING};
use crate::engine::CheckpointLog;
use crate::plugin::{
HookFuture, HookWork, InvocationEndInfo, InvocationInfo, OperationChangeInfo,
OperationInfo, Plugin, PluginFactory,
};
use crate::{BoxError, DurableContext, InvocationStatus, Options, WireError};
const ARN: &str = "arn:aws:lambda:us-east-1:1:function:test";
const REQUEST_ID: &str = "req-1";
#[derive(Clone)]
enum Answer {
Pass,
Tokenless,
TokenlessWith(Vec<Operation>, String),
Fail(ExecutionClientErrorScope),
}
type Script = dyn Fn(usize, &[OperationUpdate]) -> (Answer, Duration) + Send + Sync;
struct WithdrawingClient {
inner: InMemoryExecutionClient,
script: Box<Script>,
calls: Mutex<Vec<Vec<OperationUpdate>>>,
get_state_calls: AtomicUsize,
get_state_fails: std::sync::atomic::AtomicBool,
}
impl WithdrawingClient {
fn new(
script: impl Fn(usize, &[OperationUpdate]) -> (Answer, Duration) + Send + Sync + 'static,
) -> Arc<Self> {
Arc::new(Self {
inner: InMemoryExecutionClient::new(Vec::new()),
script: Box::new(script),
calls: Mutex::new(Vec::new()),
get_state_calls: AtomicUsize::new(0),
get_state_fails: std::sync::atomic::AtomicBool::new(false),
})
}
fn tokenless_at(n: usize) -> Arc<Self> {
Self::new(move |call, _| {
if call == n {
(Answer::Tokenless, Duration::ZERO)
} else {
(Answer::Pass, Duration::ZERO)
}
})
}
fn calls(&self) -> Vec<Vec<OperationUpdate>> {
self.calls.lock().unwrap().clone()
}
fn updates(&self) -> Vec<OperationUpdate> {
self.calls().into_iter().flatten().collect()
}
}
impl ExecutionClient for WithdrawingClient {
fn checkpoint(
&self,
execution_arn: &str,
checkpoint_token: &str,
updates: Vec<OperationUpdate>,
) -> std::pin::Pin<
Box<dyn Future<Output = Result<CheckpointOutput, ExecutionClientError>> + Send + '_>,
> {
let number = {
let mut calls = self.calls.lock().unwrap();
calls.push(updates.clone());
calls.len()
};
let (answer, delay) = (self.script)(number, &updates);
let pass = matches!(answer, Answer::Pass).then(|| {
self.inner
.checkpoint(execution_arn, checkpoint_token, updates)
});
Box::pin(async move {
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
match answer {
Answer::Pass => match pass {
Some(pass) => pass.await,
None => unreachable!("a pass answer has a future"),
},
Answer::Tokenless => Ok(CheckpointOutput::new(None, vec![], None)),
Answer::TokenlessWith(operations, marker) => {
Ok(CheckpointOutput::new(None, operations, Some(marker)))
}
Answer::Fail(scope) => Err(ExecutionClientError::new(
scope,
"scripted checkpoint failure",
)),
}
})
}
fn get_state(
&self,
execution_arn: &str,
checkpoint_token: &str,
marker: Option<&str>,
) -> std::pin::Pin<
Box<dyn Future<Output = Result<GetStateOutput, ExecutionClientError>> + Send + '_>,
> {
self.get_state_calls.fetch_add(1, Ordering::SeqCst);
if self.get_state_fails.load(Ordering::SeqCst) {
return Box::pin(async {
Err(ExecutionClientError::new_non_retryable(
"scripted get_state failure",
))
});
}
self.inner
.get_state(execution_arn, checkpoint_token, marker)
}
}
#[derive(Default)]
struct Seen {
ends: Vec<(InvocationStatus, Option<WireError>)>,
operation_starts: usize,
operation_ends: usize,
changes: usize,
}
#[derive(Clone, Default)]
struct RecordingFactory(Arc<Mutex<Seen>>);
impl RecordingFactory {
fn end(&self) -> (InvocationStatus, Option<WireError>) {
let seen = self.0.lock().unwrap();
assert_eq!(seen.ends.len(), 1, "one end hook");
seen.ends.first().cloned().unwrap()
}
}
impl PluginFactory for RecordingFactory {
fn create_plugin(
&self,
_info: &InvocationInfo,
) -> Result<Option<Box<dyn Plugin>>, BoxError> {
Ok(Some(Box::new(RecordingPlugin(Arc::clone(&self.0)))))
}
}
struct RecordingPlugin(Arc<Mutex<Seen>>);
impl Plugin for RecordingPlugin {
fn on_operation_start(&self, _info: &OperationInfo) -> HookWork {
self.0.lock().unwrap().operation_starts += 1;
Ok(None)
}
fn on_operation_end(&self, _info: &OperationInfo) -> HookWork {
self.0.lock().unwrap().operation_ends += 1;
Ok(None)
}
fn on_operation_change(&self, _info: &OperationChangeInfo) -> HookWork {
self.0.lock().unwrap().changes += 1;
Ok(None)
}
fn on_invocation_end<'a>(&'a self, info: &'a InvocationEndInfo) -> HookFuture<'a> {
self.0
.lock()
.unwrap()
.ends
.push((info.status(), info.execution_error().cloned()));
Box::pin(std::future::ready(Ok(())))
}
}
async fn run<F, Fut>(
client: &Arc<WithdrawingClient>,
factory: &RecordingFactory,
handler: F,
) -> serde_json::Value
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<u32, BoxError>> + Send,
{
run_on(client, factory, &[], handler).await
}
async fn run_on<F, Fut>(
client: &Arc<WithdrawingClient>,
factory: &RecordingFactory,
records: &[serde_json::Value],
handler: F,
) -> serde_json::Value
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<u32, BoxError>> + Send,
{
run_result(client, factory, records, handler)
.await
.expect("the invocation answers a response")
}
async fn run_result<F, Fut>(
client: &Arc<WithdrawingClient>,
factory: &RecordingFactory,
records: &[serde_json::Value],
handler: F,
) -> Result<serde_json::Value, lambda_runtime::Error>
where
F: Fn(serde_json::Value, DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<u32, BoxError>> + Send,
{
let mut operations = vec![serde_json::json!({
"Id": "root",
"Type": "EXECUTION",
"Status": "STARTED",
"ExecutionDetails": { "InputPayload": "{}" }
})];
operations.extend_from_slice(records);
let options = Options::builder()
.execution_client(Arc::clone(client) as Arc<dyn ExecutionClient>)
.plugin(factory.clone())
.build()
.unwrap();
let service = crate::wrap(handler, options);
let mut lambda_ctx = lambda_runtime::Context::default();
lambda_ctx.request_id = REQUEST_ID.to_owned();
let event = lambda_runtime::LambdaEvent::new(
crate::InvocationPayload(serde_json::json!({
"DurableExecutionArn": ARN,
"CheckpointToken": "t0",
"InitialExecutionState": { "Operations": operations }
})),
lambda_ctx,
);
tokio::time::timeout(Duration::from_secs(60), service(event))
.await
.expect("the invocation ends")
.map(|response| response.0)
}
fn last_request_has(client: &WithdrawingClient, name: &str, action: &OperationAction) {
let calls = client.calls();
let last = calls.last().expect("a request was sent");
assert!(has(last, name, action), "{calls:?}");
}
fn assert_failure_answer(
scope: ExecutionClientErrorScope,
response: Result<serde_json::Value, lambda_runtime::Error>,
factory: &RecordingFactory,
) {
let (status, error) = factory.end();
let error = error.expect("the end carries the failure");
if scope == ExecutionClientErrorScope::Invocation {
let err = response.expect_err("the invocation fails");
assert_eq!(status, InvocationStatus::Retrying);
assert_eq!(
error.error_type(),
Some(crate::error::CHECKPOINT_INVOCATION_FAILED_ERROR_TYPE)
);
assert_eq!(error.error_message(), Some(err.to_string().as_str()));
} else {
let response = response.expect("an envelope");
assert_eq!(
response
.pointer("/Status")
.and_then(serde_json::Value::as_str),
Some("FAILED"),
"{response}"
);
assert_eq!(status, InvocationStatus::Failed);
assert_eq!(
error.error_type(),
Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
assert_eq!(
response
.pointer("/Error/ErrorType")
.and_then(serde_json::Value::as_str),
Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
);
}
assert!(
error
.error_message()
.is_some_and(|m| m.contains("scripted checkpoint failure")),
"{error:?}"
);
}
fn pending() -> serde_json::Value {
serde_json::json!({ "Status": "PENDING" })
}
fn has(updates: &[OperationUpdate], name: &str, action: &OperationAction) -> bool {
updates
.iter()
.any(|u| u.name() == Some(name) && u.action() == action)
}
fn no_fail(client: &WithdrawingClient) -> bool {
client
.updates()
.iter()
.all(|u| u.action() != &OperationAction::Fail)
}
fn root_ctx(client: &Arc<WithdrawingClient>) -> DurableContext {
DurableContext::new_root_with_client(
ARN.to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(client) as Arc<dyn ExecutionClient>,
"t0".to_owned(),
)
}
fn step_start(id: &str) -> OperationUpdate {
OperationUpdate::builder()
.id(id)
.r#type(OperationType::Step)
.action(OperationAction::Start)
.build()
.unwrap()
}
#[tokio::test]
async fn tokenless_single_checkpoint_answers_pending() {
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
Ok(ctx.step(|_| async { Ok(1_u32) }).name("a").await?)
})
.await;
assert_eq!(response, pending());
assert_eq!(client.calls().len(), 1);
assert!(no_fail(&client));
assert_eq!(factory.end(), (InvocationStatus::Pending, None));
}
#[tokio::test]
async fn tokenless_buffered_checkpoint_answers_pending() {
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
let a = ctx.step(|_| async { Ok(1_u32) }).name("a").spawn();
let b = ctx.step(|_| async { Ok(2_u32) }).name("b").spawn();
Ok(a.await? + b.await?)
})
.await;
assert_eq!(response, pending());
let calls = client.calls();
assert_eq!(calls.len(), 1, "{calls:?}");
let first = calls.first().unwrap();
assert!(has(first, "a", &OperationAction::Start), "{first:?}");
assert!(has(first, "b", &OperationAction::Start), "{first:?}");
assert!(no_fail(&client));
assert_eq!(factory.end(), (InvocationStatus::Pending, None));
}
#[tokio::test(start_paused = true)]
async fn tokenless_latch_blocks_later_checkpoints() {
let client = WithdrawingClient::new(|n, _| {
if n == 1 {
(Answer::Tokenless, Duration::from_secs(1))
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
let a = ctx.step(|_| async { Ok(1_u32) }).name("a").spawn();
tokio::time::sleep(Duration::from_millis(10)).await;
let b = ctx.step(|_| async { Ok(2_u32) }).name("b").spawn();
Ok(a.await? + b.await?)
})
.await;
assert_eq!(response, pending());
let calls = client.calls();
assert_eq!(calls.len(), 1, "{calls:?}");
assert!(has(calls.first().unwrap(), "a", &OperationAction::Start));
assert!(!has(&client.updates(), "b", &OperationAction::Start));
}
#[tokio::test]
async fn tokenless_emits_lifecycle_events_not_plugin_hooks() {
let (messages, _guard) = EventMessages::install();
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
Ok(ctx
.step(|_| async { Ok(1_u32) })
.name("a")
.semantics(crate::StepSemantics::AtMostOncePerRetry)
.await?)
})
.await;
assert_eq!(response, pending());
let started = messages
.at(tracing::Level::INFO)
.into_iter()
.chain(messages.at(tracing::Level::DEBUG))
.filter(|m| m == crate::observability::event_names::OPERATION_STARTED)
.count();
assert_eq!(started, 1, "the accepted START emits its event");
let seen = factory.0.lock().unwrap();
assert_eq!(seen.changes, 0, "no on_operation_change");
assert_eq!(seen.operation_starts, 0, "no operation start hook");
assert_eq!(seen.operation_ends, 0, "no operation end hook");
}
type Recorded = (tracing::Level, String, Vec<(String, String)>);
#[derive(Clone, Default)]
struct FieldsLayer(Arc<Mutex<Vec<Recorded>>>);
struct FieldsVisitor(Vec<(String, String)>);
impl tracing::field::Visit for FieldsVisitor {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.0.push((field.name().to_owned(), format!("{value:?}")));
}
}
impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for FieldsLayer {
fn on_event(
&self,
event: &tracing::Event<'_>,
_ctx: tracing_subscriber::layer::Context<'_, S>,
) {
let mut visitor = FieldsVisitor(Vec::new());
event.record(&mut visitor);
let meta = event.metadata();
self.0
.lock()
.unwrap()
.push((*meta.level(), meta.target().to_owned(), visitor.0));
}
}
#[tokio::test]
async fn tokenless_warns_once_with_fields() {
use tracing_subscriber::layer::SubscriberExt as _;
let layer = FieldsLayer::default();
let _guard = test_tracing::install_test_subscriber(
tracing_subscriber::registry().with(layer.clone()),
);
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
drop(ctx.step(|_| async { Ok(1_u32) }).name("a").spawn());
Ok(1)
})
.await;
assert_eq!(response, pending());
let warns: Vec<Recorded> = layer
.0
.lock()
.unwrap()
.iter()
.filter(|(level, _, _)| *level == tracing::Level::WARN)
.cloned()
.collect();
assert_eq!(warns.len(), 1, "{warns:?}");
let (_, target, fields) = warns.first().unwrap();
assert_eq!(target, "aws_durable_execution_sdk::coordinator");
let field = |name: &str| {
fields
.iter()
.find(|(n, _)| n == name)
.map(|(_, v)| v.clone())
};
assert_eq!(field("executionArn").as_deref(), Some(ARN));
assert_eq!(field("requestId").as_deref(), Some(REQUEST_ID));
assert_eq!(field("message").as_deref(), Some(WITHDRAWAL_WARNING));
}
#[tokio::test(start_paused = true)]
async fn tokenless_discards_new_state() {
let state = Operation::builder()
.id("other")
.r#type(OperationType::Step)
.status(WireStatus::Succeeded)
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.build()
.unwrap();
let client = WithdrawingClient::new(move |_, _| {
(
Answer::TokenlessWith(vec![state.clone()], "marker".to_owned()),
Duration::ZERO,
)
});
let ctx = root_ctx(&client);
let write = ctx.submit_checkpoint_updates(vec![step_start("x")]);
assert!(
tokio::time::timeout(Duration::from_secs(5), write)
.await
.is_err(),
"the caller never resolves"
);
assert!(ctx.coordinator().withdrawn());
assert!(
ctx.checkpoint_status_view("other").is_none()
&& ctx.checkpoint_status_view("x").is_none(),
"the state is not merged"
);
assert_eq!(client.get_state_calls.load(Ordering::SeqCst), 0);
}
#[tokio::test(start_paused = true)]
async fn tokenless_leaves_queued_contributors_unresolved() {
let client = WithdrawingClient::new(|n, _| {
if n == 1 {
(Answer::Tokenless, Duration::from_secs(1))
} else {
(Answer::Pass, Duration::ZERO)
}
});
let ctx = DurableContext::new_root_with_client_and_coalescer(
ARN.to_owned(),
lambda_runtime::Context::default(),
Arc::new(CheckpointLog::empty()),
Arc::clone(&client) as Arc<dyn ExecutionClient>,
"t0".to_owned(),
crate::checkpoint_coalescer::CheckpointCoalescer::new(Duration::ZERO),
);
let first = ctx.submit_checkpoint_updates(vec![step_start("x")]);
while client.calls().is_empty() {
tokio::task::yield_now().await;
}
let second = ctx.submit_checkpoint_updates(vec![step_start("y")]);
let (first, second) = tokio::join!(
tokio::time::timeout(Duration::from_secs(5), first),
tokio::time::timeout(Duration::from_secs(5), second),
);
assert!(first.is_err(), "the in-flight caller never resolves");
assert!(second.is_err(), "the queued caller never resolves");
assert!(ctx.coordinator().withdrawn());
assert_eq!(client.calls().len(), 1, "the queued update is never sent");
}
#[tokio::test(start_paused = true)]
async fn withdrawal_drops_queued_and_held_writes() {
let client = WithdrawingClient::new(|n, _| {
if n == 1 {
(Answer::Tokenless, Duration::from_secs(1))
} else {
(Answer::Pass, Duration::ZERO)
}
});
let ctx = root_ctx(&client);
let first = ctx.submit_checkpoint_updates(vec![step_start("x")]);
while client.calls().is_empty() {
tokio::task::yield_now().await;
}
let held = ctx.reserve_checkpoint_updates(vec![step_start("y")]);
assert_eq!(ctx.coordinator().permits(PermitKind::Checkpoint), 2);
assert!(
tokio::time::timeout(Duration::from_secs(5), first)
.await
.is_err(),
"the in-flight caller never resolves"
);
assert!(ctx.coordinator().withdrawn());
assert_eq!(ctx.coordinator().phase(), Phase::Suspending);
assert_eq!(
ctx.coordinator().permits(PermitKind::Checkpoint),
0,
"the held write was dropped with its permit"
);
let held = held.release();
assert!(
tokio::time::timeout(Duration::from_secs(5), held)
.await
.is_err(),
"the held caller never resolves"
);
assert_eq!(client.calls().len(), 1, "the held write is never sent");
}
async fn queued_client_error(
scope: ExecutionClientErrorScope,
) -> (
Arc<WithdrawingClient>,
Result<serde_json::Value, lambda_runtime::Error>,
RecordingFactory,
) {
let client = WithdrawingClient::new(move |_, updates| {
if has(updates, "a", &OperationAction::Start) {
(Answer::Fail(scope), Duration::ZERO)
} else if has(updates, "b", &OperationAction::Start) {
(Answer::Tokenless, Duration::ZERO)
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run_result(&client, &factory, &[], |_event, ctx| async move {
let a = ctx.step(|_| async { Ok(1_u32) }).name("a").spawn();
let b = ctx.step(|_| async { Ok(2_u32) }).name("b").spawn();
Ok(a.await? + b.await?)
})
.await;
(client, response, factory)
}
#[tokio::test]
async fn queued_write_behind_invocation_client_error_is_not_sent() {
let scope = ExecutionClientErrorScope::Invocation;
let (client, response, factory) = queued_client_error(scope).await;
assert_eq!(client.calls().len(), 1, "only a's START");
assert_failure_answer(scope, response, &factory);
}
#[tokio::test]
async fn queued_write_behind_execution_client_error_is_not_sent() {
let scope = ExecutionClientErrorScope::Execution;
let (client, response, factory) = queued_client_error(scope).await;
assert_eq!(client.calls().len(), 1, "only a's START");
assert!(no_fail(&client), "no terminal FAIL is sent");
assert_failure_answer(scope, response, &factory);
}
fn tokenless_start_of_a_after_a_second() -> Arc<WithdrawingClient> {
WithdrawingClient::new(|_, updates| {
if has(updates, "a", &OperationAction::Start) {
(Answer::Tokenless, Duration::from_secs(1))
} else {
(Answer::Pass, Duration::ZERO)
}
})
}
#[tokio::test]
async fn pending_wins_over_handler_ok() {
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
drop(ctx.step(|_| async { Ok(1_u32) }).name("a").spawn());
Ok(1)
})
.await;
assert_eq!(response, pending());
assert!(
client
.updates()
.iter()
.all(|u| u.r#type() != &OperationType::Execution),
"no EXECUTION update is sent"
);
assert_eq!(factory.end(), (InvocationStatus::Pending, None));
}
#[tokio::test]
async fn pending_wins_over_handler_err() {
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
drop(ctx.step(|_| async { Ok(1_u32) }).name("a").spawn());
Err::<u32, BoxError>("boom".into())
})
.await;
assert_eq!(response, pending());
let (status, error) = factory.end();
assert_eq!(status, InvocationStatus::Pending);
assert_eq!(error.unwrap().error_message(), Some("boom"));
}
async fn failed_after_handler(
handler_err: bool,
) -> (
Arc<WithdrawingClient>,
Result<serde_json::Value, lambda_runtime::Error>,
RecordingFactory,
) {
let client = WithdrawingClient::new(|_, updates| {
if has(updates, "a", &OperationAction::Start) {
(
Answer::Fail(ExecutionClientErrorScope::Invocation),
Duration::from_secs(1),
)
} else if has(updates, "b", &OperationAction::Start) {
(Answer::Tokenless, Duration::ZERO)
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run_result(&client, &factory, &[], move |_event, ctx| async move {
drop(ctx.step(|_| async { Ok(1_u32) }).name("a").spawn());
drop(ctx.step(|_| async { Ok(2_u32) }).name("b").spawn());
if handler_err {
return Err::<u32, BoxError>("boom".into());
}
Ok(1)
})
.await;
(client, response, factory)
}
#[tokio::test(start_paused = true)]
async fn failure_after_handler_ok_decides_and_stops_later_writes() {
let (client, response, factory) = failed_after_handler(false).await;
assert_eq!(client.calls().len(), 1, "only a's START");
assert_failure_answer(ExecutionClientErrorScope::Invocation, response, &factory);
}
#[tokio::test(start_paused = true)]
async fn failure_after_handler_err_decides_and_stops_later_writes() {
let (client, response, factory) = failed_after_handler(true).await;
assert_eq!(client.calls().len(), 1, "only a's START");
assert_failure_answer(ExecutionClientErrorScope::Invocation, response, &factory);
}
#[tokio::test(start_paused = true)]
async fn failure_of_stopped_caller_decides_and_stops_later_writes() {
let client = WithdrawingClient::new(|_, updates| {
if has(updates, "a", &OperationAction::Start) {
(
Answer::Fail(ExecutionClientErrorScope::Invocation),
Duration::from_secs(1),
)
} else if has(updates, "w", &OperationAction::Start) {
(Answer::Tokenless, Duration::ZERO)
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run_result(&client, &factory, &[], |_event, ctx| async move {
ctx.run_in_child_context(|child| async move {
drop(child.step(|_| async { Ok(1_u32) }).name("a").spawn());
tokio::time::sleep(Duration::from_millis(500)).await;
Ok(0_u32)
})
.name("c")
.await?;
ctx.wait(Duration::from_secs(60)).name("w").await?;
Ok(1)
})
.await;
assert!(!has(&client.updates(), "w", &OperationAction::Start));
last_request_has(&client, "a", &OperationAction::Start);
assert_failure_answer(ExecutionClientErrorScope::Invocation, response, &factory);
}
async fn stopped_write_failure(
handler_err: bool,
) -> (
Arc<WithdrawingClient>,
Result<serde_json::Value, lambda_runtime::Error>,
RecordingFactory,
) {
let client = WithdrawingClient::new(|_, updates| {
if has(updates, "a", &OperationAction::Succeed) {
(
Answer::Fail(ExecutionClientErrorScope::Execution),
Duration::from_secs(1),
)
} else if has(updates, "a", &OperationAction::Fail) {
(Answer::Tokenless, Duration::from_secs(2))
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run_result(&client, &factory, &[], move |_event, ctx| async move {
drop(ctx.step(|_| async { Ok(1_u32) }).name("a").spawn());
tokio::time::sleep(Duration::from_millis(1500)).await;
if handler_err {
return Err::<u32, BoxError>("boom".into());
}
Ok(1)
})
.await;
(client, response, factory)
}
#[tokio::test(start_paused = true)]
async fn failed_outcome_write_decides_over_handler_ok_and_sends_no_fail() {
let (client, response, factory) = stopped_write_failure(false).await;
assert!(no_fail(&client), "a's terminal FAIL is not sent");
last_request_has(&client, "a", &OperationAction::Succeed);
assert_failure_answer(ExecutionClientErrorScope::Execution, response, &factory);
}
#[tokio::test(start_paused = true)]
async fn failed_outcome_write_decides_over_handler_err_and_sends_no_fail() {
let (client, response, factory) = stopped_write_failure(true).await;
assert!(no_fail(&client), "a's terminal FAIL is not sent");
last_request_has(&client, "a", &OperationAction::Succeed);
assert_failure_answer(ExecutionClientErrorScope::Execution, response, &factory);
}
async fn earlier_client_error(
scope: ExecutionClientErrorScope,
) -> (
Arc<WithdrawingClient>,
Result<serde_json::Value, lambda_runtime::Error>,
RecordingFactory,
) {
let client = WithdrawingClient::new(move |_, updates| {
if has(updates, "a", &OperationAction::Succeed) {
(Answer::Fail(scope), Duration::from_secs(1))
} else if has(updates, "b", &OperationAction::Start) {
(Answer::Tokenless, Duration::from_secs(1))
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run_result(&client, &factory, &[], |_event, ctx| async move {
let a = ctx.step(|_| async { Ok(1_u32) }).name("a").spawn();
tokio::time::sleep(Duration::from_millis(500)).await;
let b = ctx.step(|_| async { Ok(2_u32) }).name("b").spawn();
Ok(a.await? + b.await?)
})
.await;
(client, response, factory)
}
#[tokio::test(start_paused = true)]
async fn invocation_client_error_stops_the_write_waiting_behind_it() {
let scope = ExecutionClientErrorScope::Invocation;
let (client, response, factory) = earlier_client_error(scope).await;
assert!(!has(&client.updates(), "b", &OperationAction::Start));
last_request_has(&client, "a", &OperationAction::Succeed);
assert_failure_answer(scope, response, &factory);
}
#[tokio::test(start_paused = true)]
async fn execution_client_error_stops_the_write_waiting_behind_it() {
let scope = ExecutionClientErrorScope::Execution;
let (client, response, factory) = earlier_client_error(scope).await;
assert!(!has(&client.updates(), "b", &OperationAction::Start));
last_request_has(&client, "a", &OperationAction::Succeed);
assert!(no_fail(&client), "no terminal FAIL is sent");
assert_failure_answer(scope, response, &factory);
}
#[tokio::test(start_paused = true)]
async fn pending_wins_over_context_validation_fatal() {
let client = tokenless_start_of_a_after_a_second();
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
let cloned = ctx.clone();
Ok(ctx
.step(move |_| async move {
let inner = cloned.step(|_| async { Ok(2_u32) }).name("inner").await?;
Ok(inner)
})
.name("a")
.await?)
})
.await;
assert_eq!(response, pending());
let (status, error) = factory.end();
assert_eq!(status, InvocationStatus::Pending);
let error = error.unwrap();
assert_eq!(error.error_type(), Some("Error"));
assert!(
error
.error_message()
.is_some_and(|m| m.starts_with("context usage error")),
"{error:?}"
);
}
struct FailingSerdes;
impl crate::serdes::Serdes<u32> for FailingSerdes {
fn serialize(
&self,
_value: u32,
_context: crate::serdes::SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Err(injected_serdes_failure()))
}
fn deserialize(
&self,
_wire: String,
_context: crate::serdes::SerdesContext,
) -> impl Future<Output = Result<u32, BoxError>> + Send {
std::future::ready(Err(injected_serdes_failure()))
}
}
fn injected_serdes_failure() -> BoxError {
Box::new(crate::serdes::RetryableSerdesError::new(
"injected serdes failure",
))
}
#[tokio::test(start_paused = true)]
async fn pending_wins_over_serdes_fault() {
let client = tokenless_start_of_a_after_a_second();
let factory = RecordingFactory::default();
let replayed = serde_json::json!({
"Id": "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b",
"Type": "STEP",
"SubType": "Step",
"Name": "r",
"Status": "SUCCEEDED",
"StepDetails": { "Attempt": 1, "Result": "1" }
});
let response = run_on(&client, &factory, &[replayed], |_event, ctx| async move {
let replayed = ctx
.step(|_| async { Ok(1_u32) })
.name("r")
.serdes(FailingSerdes)
.future();
drop(ctx.step(|_| async { Ok(2_u32) }).name("a").spawn());
Ok(replayed.await?)
})
.await;
assert!(has(&client.updates(), "a", &OperationAction::Start));
assert_eq!(response, pending());
let (status, error) = factory.end();
assert_eq!(status, InvocationStatus::Pending);
let error = error.unwrap();
assert_eq!(
error.error_type(),
Some(crate::error::SERDES_FAILED_ERROR_TYPE)
);
assert!(
error
.error_message()
.is_some_and(|m| m.contains("injected serdes failure")),
"{error:?}"
);
}
#[tokio::test]
async fn pure_withdrawal_end_hook_has_no_error() {
let client = WithdrawingClient::tokenless_at(1);
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
ctx.step(|_| async { Ok(1_u32) }).name("a").await?;
std::future::pending::<()>().await;
Ok(1)
})
.await;
assert_eq!(response, pending());
assert_eq!(factory.end(), (InvocationStatus::Pending, None));
}
fn execution_update(action: &OperationAction) -> OperationUpdate {
let builder = OperationUpdate::builder()
.id("root")
.r#type(OperationType::Execution)
.action(action.clone());
let builder = if *action == OperationAction::Fail {
builder.error(
aws_sdk_lambda::types::ErrorObject::builder()
.error_type("Boom")
.error_message("boom")
.build(),
)
} else {
builder.payload("1")
};
builder.build().unwrap()
}
#[tokio::test]
async fn execution_update_exempt_from_withdrawal() {
for (action, status) in [
(OperationAction::Succeed, WireStatus::Succeeded),
(OperationAction::Fail, WireStatus::Failed),
] {
let (messages, _guard) = EventMessages::install();
let terminal = Operation::builder()
.id("root")
.r#type(OperationType::Execution)
.status(status.clone())
.start_timestamp(aws_smithy_types::DateTime::from_secs(0))
.build()
.unwrap();
let client = WithdrawingClient::new(move |_, _| {
(
Answer::TokenlessWith(vec![terminal.clone()], String::new()),
Duration::ZERO,
)
});
let ctx = root_ctx(&client);
let output = tokio::time::timeout(
Duration::from_secs(5),
ctx.checkpoint_updates_direct(vec![execution_update(&action)]),
)
.await
.expect("the call resolves")
.expect("the call returns Ok");
assert_eq!(output.checkpoint_token(), None);
assert!(
output
.updated_operations
.iter()
.any(|op| op.r#type() == &OperationType::Execution && op.status() == &status),
"the execution is terminal"
);
assert!(!ctx.coordinator().withdrawn(), "no latch is set");
assert_eq!(ctx.coordinator().phase(), Phase::Running);
assert_eq!(messages.at(tracing::Level::WARN), Vec::<String>::new());
}
}
#[tokio::test]
async fn execution_update_exempt_skips_pagination() {
for action in [OperationAction::Succeed, OperationAction::Fail] {
let client = WithdrawingClient::new(move |_, _| {
(
Answer::TokenlessWith(Vec::new(), "marker".to_owned()),
Duration::ZERO,
)
});
client.get_state_fails.store(true, Ordering::SeqCst);
let ctx = root_ctx(&client);
let output = tokio::time::timeout(
Duration::from_secs(5),
ctx.checkpoint_updates_direct(vec![execution_update(&action)]),
)
.await
.expect("the call resolves")
.expect("the call returns Ok");
assert_eq!(output.checkpoint_token(), None);
assert!(!ctx.coordinator().withdrawn(), "no latch is set");
assert_eq!(ctx.coordinator().phase(), Phase::Running);
assert_eq!(
client.get_state_calls.load(Ordering::SeqCst),
0,
"{action:?}: no get_state call"
);
}
}
#[tokio::test(start_paused = true)]
async fn refresh_checkpoint_withdrawn() {
let client = WithdrawingClient::new(|_, updates| {
if updates.is_empty() {
(Answer::Tokenless, Duration::ZERO)
} else {
(Answer::Pass, Duration::ZERO)
}
});
let factory = RecordingFactory::default();
let response = run(&client, &factory, |_event, ctx| async move {
let wait = ctx.wait(Duration::from_secs(1)).name("w").spawn();
let step = ctx
.step(|_| async {
tokio::time::sleep(Duration::from_secs(5)).await;
Ok(1_u32)
})
.name("s")
.spawn();
let (waited, value) = tokio::join!(wait, step);
waited?;
Ok(value?)
})
.await;
assert_eq!(response, pending());
let calls = client.calls();
let refresh = calls
.iter()
.position(Vec::is_empty)
.expect("a refresh was sent");
assert_eq!(
refresh + 1,
calls.len(),
"no checkpoint after the refresh: {calls:?}"
);
assert!(!has(&client.updates(), "s", &OperationAction::Succeed));
assert_eq!(factory.end(), (InvocationStatus::Pending, None));
}
}