Skip to main content

aws_durable_execution_sdk/
step.rs

1//! Step operation execution engine.
2//!
3//! Implements the live path (run closure, serialize, checkpoint), replay path
4//! (return frozen result), and retry strategy (backend-owned retry timer).
5//!
6//! Retry delays use the backend's timer rather than an in-process sleep:
7//! a RETRY action with `NextAttemptDelaySeconds` is checkpointed, then the
8//! step waits for its record to leave `PENDING`. When the record turns
9//! `READY` while other work runs, the next attempt runs in the same
10//! invocation; otherwise the invocation suspends.
11
12use std::future::Future;
13use std::time::Duration;
14
15use aws_sdk_lambda::types::{OperationAction, OperationType, OperationUpdate};
16
17use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
18use crate::context::{DurableContext, StepContext, SubmittedWrite};
19use crate::engine::{CheckpointStatus, OperationId};
20use crate::error::{
21    OperationError, OperationErrorKind, STEP_INTERRUPTED_MESSAGE, StepError, StepErrorKind,
22};
23use crate::plugin::UserBody;
24use crate::serdes::SerdesContext;
25use crate::start::StartStep;
26use crate::tracing_layer;
27use crate::{BoxError, OperationSubType, RetryDecision, RetryStrategy, Serdes};
28
29/// The default retry strategy of a step, a `with_retry` block and a
30/// `wait_for_callback` submitter: 6 total attempts, 5s initial delay, 60s
31/// max delay, 2x backoff rate, FULL jitter.
32///
33/// The constants live in [`RetryStrategyConfig::default_preset`], the JS
34/// `default` preset; this is one call so they are defined in exactly one
35/// place.
36pub(crate) fn default_retry_strategy() -> RetryStrategy {
37    RetryStrategyConfig::default_preset().into_retry_strategy()
38}
39
40impl RetryStrategyConfig {
41    /// Computes the retry decision for `err` at a 1-based failed attempt
42    /// number.
43    ///
44    /// The checks run in the JS order: the attempt cap first (the error
45    /// filter is not called once `attempt` reaches `max_attempts`), then
46    /// the error filter set with
47    /// [`retry_if`](crate::builders::RetryStrategyConfigBuilder::retry_if),
48    /// then the delay schedule.
49    ///
50    /// The base delay follows the [`Backoff`]:
51    /// `initial_delay * rate^(attempt - 1)` for an exponential backoff, and
52    /// `initial_delay + increment * (attempt - 1)` for a linear one. Both are
53    /// capped at `max_delay`, jittered per the configured
54    /// [`JitterStrategy`], and rounded to the nearest whole second with a
55    /// one-second minimum for every jitter mode (see [`quantize_delay`]).
56    /// Stops once `attempt` reaches `max_attempts`.
57    pub(crate) fn decide(&self, err: &StepError, attempt: u32) -> RetryDecision {
58        if attempt >= self.max_attempts() {
59            return RetryDecision::Stop;
60        }
61        if let Some(predicate) = self.retry_predicate()
62            && !predicate(err)
63        {
64            return RetryDecision::Stop;
65        }
66
67        // attempt is 1-based: first failure is attempt=1.
68        let initial = self.initial_delay().as_secs_f64();
69        let base = match self.backoff() {
70            // Exponential backoff: initial * rate^(attempt-1).
71            Backoff::Exponential { rate } => {
72                let exponent = (i32::try_from(attempt).unwrap_or(1)) - 1;
73                initial * rate.powi(exponent)
74            }
75            // Linear backoff: initial + increment * (attempt-1).
76            Backoff::Linear { increment } => {
77                initial + increment.as_secs_f64() * f64::from(attempt.saturating_sub(1))
78            }
79        }
80        .min(self.max_delay().as_secs_f64());
81
82        let jittered = match self.jitter() {
83            JitterStrategy::None => base,
84            // Half jitter: base/2 plus random in [0, base/2] => [base/2, base].
85            JitterStrategy::Half => base / 2.0 + rand_full_jitter(base / 2.0),
86            // Full jitter: random in [0, base].
87            JitterStrategy::Full => rand_full_jitter(base),
88        };
89
90        RetryDecision::retry(quantize_delay(jittered))
91    }
92
93    /// Converts this configuration into the boxed retry-strategy closure
94    /// the step engine consumes. The error reaches only the error filter;
95    /// the delay depends only on the attempt number.
96    pub(crate) fn into_retry_strategy(self) -> RetryStrategy {
97        Box::new(move |err: &StepError, attempt: u32| self.decide(err, attempt))
98    }
99}
100
101/// Full jitter: returns a value in `[0, max_secs]`.
102///
103/// Uses time + thread-id + counter hashing for determinism-safe randomness
104/// (no rand crate dependency).
105pub(crate) fn rand_full_jitter(max_secs: f64) -> f64 {
106    use std::collections::hash_map::DefaultHasher;
107    use std::hash::{Hash, Hasher};
108    use std::time::SystemTime;
109
110    static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
111
112    let mut hasher = DefaultHasher::new();
113    SystemTime::now().hash(&mut hasher);
114    std::thread::current().id().hash(&mut hasher);
115    // Mix in a counter to avoid same result on rapid calls.
116    COUNTER
117        .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
118        .hash(&mut hasher);
119    let bits = hasher.finish();
120    // Map u64 to [0.0, 1.0)
121    #[expect(clippy::cast_precision_loss)] // reason: approximation is fine for jitter
122    let fraction = (bits as f64) / (u64::MAX as f64);
123    fraction * max_secs
124}
125
126/// Rounds a delay in seconds to the nearest whole second, minimum 1.
127/// A non-finite input returns 1 second.
128///
129/// Both strategy paths ([`RetryStrategyConfig`] and the wait strategy of
130/// `wait_for_condition`) quantize a computed delay with this function, for
131/// every jitter mode. A delay a caller gives directly (a custom closure or
132/// `ctx.wait`) rounds up instead, so it never ends early.
133pub(crate) fn quantize_delay(secs: f64) -> Duration {
134    Duration::try_from_secs_f64(secs.round().max(1.0)).unwrap_or(Duration::from_secs(1))
135}
136
137/// Execution semantics for a step, controlling behavior on interrupted replay.
138///
139/// When a step is interrupted mid-execution (e.g., a Lambda timeout or crash),
140/// the checkpoint log records a `Started` status without an outcome. On the
141/// next invocation, the SDK must decide whether to re-execute the step body.
142///
143/// This is a **client-side only** configuration: it is not sent on the wire.
144/// The checkpoint `Start` action is identical regardless of semantics; the
145/// difference is purely in the replay decision when `Started` status is
146/// encountered.
147///
148/// # Examples
149///
150/// ```
151/// use aws_durable_execution_sdk::StepSemantics;
152///
153/// // Default: re-execute on replay if interrupted.
154/// let default = StepSemantics::AtLeastOncePerRetry;
155///
156/// // Idempotency-sensitive: treat interrupted as failed, consult retry strategy.
157/// let at_most_once = StepSemantics::AtMostOncePerRetry;
158/// # drop(default);
159/// # drop(at_most_once);
160/// ```
161#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
162#[non_exhaustive]
163pub enum StepSemantics {
164    /// Re-execute the step body on replay if the previous attempt was
165    /// interrupted (status `Started` with no outcome).
166    ///
167    /// This is the default behavior and matches the semantics of most
168    /// idempotent operations. The step may execute more than once per retry
169    /// attempt if the process is interrupted between `Start` checkpoint and
170    /// the outcome checkpoint.
171    ///
172    /// The body starts without waiting for the `START` checkpoint. The
173    /// outcome is written only after `START` is acknowledged; if `START`
174    /// fails, the outcome is not written and the invocation fails.
175    #[default]
176    AtLeastOncePerRetry,
177
178    /// Do **not** re-execute the step body if the previous attempt was
179    /// interrupted. Instead, treat the interruption as a failure and
180    /// consult the retry strategy to decide whether to schedule a new
181    /// attempt or fail permanently.
182    ///
183    /// The retry strategy sees [`StepErrorKind::Interrupted`], and the
184    /// RETRY or FAIL record carries the error type `StepInterruptedError`.
185    /// When the strategy stops, the step returns that kind, live and on
186    /// replay.
187    ///
188    /// Use this for non-idempotent operations (e.g., charging a credit
189    /// card) where re-execution could produce duplicate side effects.
190    AtMostOncePerRetry,
191}
192
193/// Wire sub-type for step operations.
194pub(crate) const STEP_SUB_TYPE: &str = "Step";
195
196/// Internal state for step execution passed from the builder.
197///
198/// Generic over the user's closure `F` (and, through `F`'s output, its
199/// future): the closure is stored and run **without type erasure**. The one
200/// erasure point for a step is `.future()` / `into_future` on the builder,
201/// which boxes the whole execution future once inside
202/// [`DurableFuture`](crate::DurableFuture).
203pub(crate) struct StepExecution<O, F, S> {
204    pub(crate) ctx: DurableContext,
205    pub(crate) op_id: OperationId,
206    pub(crate) name: Option<String>,
207    pub(crate) retry_strategy: Option<RetryStrategy>,
208    pub(crate) serdes: S,
209    pub(crate) semantics: StepSemantics,
210    /// The resolved subtype: the builder's `sub_type`, or `Step`.
211    pub(crate) sub_type: OperationSubType,
212    /// The name the plugin hooks report instead of `name`: the submitter
213    /// step of a named `wait_for_callback` reports `<name>-submitter`.
214    /// `None` reports `name`.
215    pub(crate) plugin_name: Option<String>,
216    pub(crate) closure: F,
217    pub(crate) _marker: std::marker::PhantomData<fn() -> O>,
218}
219
220impl<O, F, Fut, S> StepExecution<O, F, S>
221where
222    O: Send + 'static,
223    F: FnOnce(StepContext) -> Fut + Clone + Send + 'static,
224    Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
225    S: Serdes<O>,
226{
227    /// Executes the step operation: one loop over its record.
228    ///
229    /// Thin generic wrapper: the ONLY code monomorphized per step call
230    /// site. The replay/checkpoint state machine lives in the non-generic
231    /// [`StepCore`] (generic over the result type `O` only); this wrapper
232    /// polls the user's concrete future between its halves, one attempt
233    /// per cycle, and waits for the retry timer between cycles.
234    ///
235    /// Each attempt calls a fresh clone of the closure. The original stays
236    /// here for the next attempt, so its captures never move into an
237    /// attempt.
238    pub(crate) async fn execute(self) -> Result<O, OperationError> {
239        let (core, closure) = self.split();
240        Self::run(core, closure, None).await
241    }
242
243    /// Runs the step's start step now, on the caller's task, and returns
244    /// the future of the rest of the step (see [`crate::start`]).
245    /// `.spawn()` uses it, so the `START` is queued before it returns.
246    pub(crate) fn execute_started(
247        self,
248    ) -> impl Future<Output = Result<O, OperationError>> + Send + 'static {
249        let (core, closure) = self.split();
250        let first = core.start();
251        Self::run(core, closure, Some(first))
252    }
253
254    /// Splits the execution into its non-generic core and the closure.
255    fn split(self) -> (StepCore<O, S>, F) {
256        let Self {
257            ctx,
258            op_id,
259            name,
260            retry_strategy,
261            serdes,
262            semantics,
263            sub_type,
264            plugin_name,
265            closure,
266            _marker,
267        } = self;
268        let core = StepCore {
269            ctx,
270            op_id,
271            name,
272            retry_strategy,
273            serdes,
274            semantics,
275            sub_type,
276            plugin_name,
277            _marker: std::marker::PhantomData,
278        };
279        (core, closure)
280    }
281
282    /// The step loop. `first` is the result of the first cycle's start
283    /// step when it already ran; `None` runs it in the first cycle.
284    async fn run(
285        core: StepCore<O, S>,
286        mut closure: F,
287        mut first: Option<Result<StartStep, OperationError>>,
288    ) -> Result<O, OperationError> {
289        // True once the step has waited for its record in this invocation.
290        // A terminal record found after that wait changed during this
291        // invocation, so it is a continuation, not a replay.
292        let mut resumed = false;
293        loop {
294            match core.before(resumed, first.take()).await? {
295                StepPrelude::Done(result) => return result,
296                StepPrelude::Wait => {}
297                StepPrelude::Run {
298                    attempt,
299                    span,
300                    start,
301                    info,
302                } => {
303                    // The step body runs as a `Step` body: an operation
304                    // claimed inside it fails the execution with a context
305                    // usage error. It holds an activity permit from its call
306                    // until it returns, so the invocation does not suspend
307                    // while it runs.
308                    let body = UserBody::Step(core.op_id.positional().to_owned());
309                    let step_ctx = StepContext::new(attempt);
310                    // `F` is `Send` but maybe not `Sync`, so the attempt
311                    // borrows it mutably: `&mut F` is `Send`. The borrow
312                    // ends when this attempt's await returns.
313                    let closure = &mut closure;
314                    // The attempt is a user function: its hooks and wrap
315                    // run through the runner; the permit stays here. The
316                    // clone runs the captures' `Clone` implementations,
317                    // which are user code, so the runner calls it inside
318                    // the wraps, the active body and the step's span.
319                    let ran = core
320                        .ctx
321                        .hold_activity(crate::plugin::run_user_function(
322                            core.ctx.shared_plugins(),
323                            info,
324                            body,
325                            span,
326                            move || {
327                                let attempt_closure = (*closure).clone();
328                                attempt_closure(step_ctx)
329                            },
330                            |error: &BoxError| {
331                                crate::error::wire_error_for(&**error, STEP_FALLBACK_ERROR_TYPE)
332                            },
333                        ))
334                        .await;
335                    // The failure keeps the runner's wire: the error's
336                    // `Display` runs once, in the runner.
337                    core.await_start(start).await?;
338                    match core.settle(attempt, ran).await? {
339                        StepSettled::Done(value) => return Ok(value),
340                        StepSettled::Retry => {}
341                    }
342                }
343            }
344            // The record is PENDING until its retry timer fires. Wait until
345            // a checkpoint response carries READY or a terminal status, as
346            // the Go SDK does, then handle the record as a new invocation
347            // would: READY runs the next attempt, a terminal status returns
348            // its outcome. The service never answers a RETRY with STARTED,
349            // so a STARTED record (a response that did not carry the
350            // RETRY's result) keeps waiting rather than starting another
351            // attempt at once. When no other SDK work runs, the invocation
352            // suspends here instead.
353            core.ctx
354                .wait_for_record(core.op_id.wire(), |status| {
355                    *status == CheckpointStatus::Ready || status.is_terminal()
356                })
357                .await;
358            resumed = true;
359        }
360    }
361}
362
363/// The non-generic half of a step: task-ownership check, replay
364/// resolution, attempt derivation, the START checkpoint, and outcome
365/// checkpointing. Generic only over the result type `O`, no user closure
366/// reaches this state machine, so its substantial replay/checkpoint logic
367/// compiles once per result type instead of once per step call site.
368struct StepCore<O, S> {
369    ctx: DurableContext,
370    op_id: OperationId,
371    name: Option<String>,
372    retry_strategy: Option<RetryStrategy>,
373    serdes: S,
374    semantics: StepSemantics,
375    sub_type: OperationSubType,
376    /// The name the plugin hooks report instead of `name`, if any.
377    plugin_name: Option<String>,
378    _marker: std::marker::PhantomData<fn() -> O>,
379}
380
381/// What [`StepCore::before`] decided for one cycle of the step loop.
382enum StepPrelude<O> {
383    /// Resolved without running the body: a recorded terminal outcome, or
384    /// an `AtMostOncePerRetry` interruption the retry strategy stopped.
385    Done(Result<O, OperationError>),
386    /// The record is `PENDING`, or a `RETRY` was just recorded for an
387    /// interrupted `AtMostOncePerRetry` attempt: wait for the retry timer.
388    Wait,
389    /// The body must run at `attempt`, inside `span`.
390    Run {
391        /// 1-based attempt number for this live execution.
392        attempt: u32,
393        /// The operation span the body runs under.
394        span: tracing::Span,
395        /// The in-flight `START` write of an at-least-once attempt, awaited
396        /// after the body returns and before the outcome is written. `None`
397        /// when START was awaited before the body, or was already recorded.
398        /// A body that panics drops it, and the write continues.
399        start: Option<SubmittedWrite>,
400        /// The attempt's info, as `on_attempt_start` saw it.
401        info: std::sync::Arc<crate::plugin::OperationInfo>,
402    },
403}
404
405/// How [`StepCore::settle`] ended one attempt.
406enum StepSettled<O> {
407    /// The step succeeded with this value.
408    Done(O),
409    /// A `RETRY` was checkpointed: the next cycle waits for its timer.
410    Retry,
411}
412
413impl<O, S> StepCore<O, S>
414where
415    O: Send + 'static,
416    S: Serdes<O>,
417{
418    /// Emits `operation_replayed` for a recorded terminal outcome, unless
419    /// the record changed during this invocation (`resumed`).
420    fn emit_replayed(&self, resumed: bool, attempt: u32) {
421        if !resumed {
422            self.ctx.emit_operation_replayed(
423                self.op_id.wire(),
424                self.name.as_deref(),
425                "Step",
426                Some(self.sub_type.as_str()),
427                attempt,
428            );
429        }
430    }
431
432    /// The START update of this step.
433    fn start_update(&self) -> OperationUpdate {
434        build_start_update(
435            self.op_id.wire(),
436            self.name.as_deref(),
437            self.sub_type.as_str(),
438            self.ctx.parent_wire_id(),
439        )
440    }
441
442    /// Runs everything that precedes the user closure in one cycle: replay
443    /// resolution, or the live-path preamble ending at the START
444    /// checkpoint.
445    ///
446    /// `resumed` is true when the step waited for its record in this
447    /// invocation. A terminal record then changed during this invocation,
448    /// so it returns its outcome without an `operation_replayed` event.
449    ///
450    /// `first` is the result of this cycle's start step when the caller
451    /// already ran it (see [`StepExecution::execute_started`]).
452    async fn before(
453        &self,
454        resumed: bool,
455        first: Option<Result<StartStep, OperationError>>,
456    ) -> Result<StepPrelude<O>, OperationError> {
457        // 1. The start step: admission, task ownership, the replay identity
458        // check, and the START of a live attempt (see `crate::start`).
459        let started = match first {
460            Some(started) => started,
461            None => self.start(),
462        }?;
463
464        let wire_id = self.op_id.wire();
465
466        // 2. A record that queues no START: replay its outcome, wait for
467        // its retry timer, or re-enter its interrupted attempt.
468        self.after_start(resumed, started, wire_id).await
469    }
470
471    /// The start step of one cycle (see [`crate::start`]).
472    fn start(&self) -> Result<StartStep, OperationError> {
473        crate::start::start_step(
474            &self.ctx,
475            &crate::start::StartSpec {
476                op_id: &self.op_id,
477                operation_type: "Step",
478                sub_type: self.sub_type.as_str(),
479                name: self.name.as_deref(),
480                order: match self.semantics {
481                    StepSemantics::AtLeastOncePerRetry => crate::start::StartOrder::QueueThenHook,
482                    StepSemantics::AtMostOncePerRetry => crate::start::StartOrder::AwaitResponse,
483                },
484            },
485            |view| self.cycle(view),
486            || self.start_update(),
487            |plugins, is_replay| self.operation_started(plugins, is_replay),
488        )
489    }
490
491    /// The rest of one cycle after its start step `started`.
492    async fn after_start(
493        &self,
494        resumed: bool,
495        started: StartStep,
496        wire_id: &str,
497    ) -> Result<StepPrelude<O>, OperationError> {
498        let start = match started {
499            StartStep::Refused => return std::future::pending().await,
500            StartStep::Queued(write) => Some(write),
501            // AtLeastOncePerRetry (default): skip re-checkpointing START but
502            // re-execute the body. The start step fired the replay start
503            // hook.
504            StartStep::Recorded(view)
505                if view.status == CheckpointStatus::Started
506                    && self.semantics == StepSemantics::AtLeastOncePerRetry =>
507            {
508                None
509            }
510            StartStep::Recorded(view) => return self.recorded(resumed, &view).await,
511        };
512
513        // 3. Live execution path.
514        // Derive the current attempt from checkpoint log: if there's a
515        // recorded operation with step details, attempt = recorded + 1.
516        let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
517
518        // An at-least-once attempt runs its body while START is in flight
519        // and awaits it before the outcome write (see `StepPrelude::Run`).
520        // An at-most-once body must not run without a recorded START, so
521        // that attempt awaits START here, then fires its start hooks.
522        let start = match start {
523            None => None,
524            Some(start) if self.semantics == StepSemantics::AtLeastOncePerRetry => Some(start),
525            Some(start) => {
526                if let Err(err) = start.await {
527                    return self.ctx.checkpoint_failure_unrecoverable(err).await;
528                }
529                if let Some(plugins) = self.ctx.plugins() {
530                    self.operation_started(plugins, attempt > 1);
531                }
532                None
533            }
534        };
535
536        // The body runs inside a tracing span carrying the structured-log
537        // field contract.
538        let is_replay = false; // Live execution is never replay.
539        let span = tracing_layer::operation_span(
540            self.ctx.execution_arn(),
541            &self.ctx.lambda_context().request_id,
542            self.ctx.lambda_context().tenant_id.as_deref(),
543            wire_id,
544            self.name.as_deref(),
545            attempt,
546            is_replay,
547        );
548
549        let info = self.attempt_started(attempt);
550
551        Ok(StepPrelude::Run {
552            attempt,
553            span,
554            start,
555            info,
556        })
557    }
558
559    /// Handles a record for which this cycle neither queues a `START` nor
560    /// re-enters an at-least-once attempt: it replays a terminal outcome,
561    /// waits for a retry timer, or fails an interrupted at-most-once
562    /// attempt.
563    async fn recorded(
564        &self,
565        resumed: bool,
566        view: &crate::engine::CheckpointStatusView,
567    ) -> Result<StepPrelude<O>, OperationError> {
568        let positional_id = self.op_id.positional();
569        let wire_id = self.op_id.wire();
570        match view.status {
571            CheckpointStatus::Succeeded => {
572                // Decode the recorded payload FIRST: `operation_replayed`
573                // promises a recorded terminal outcome was returned, so a
574                // corrupt payload or failing serdes surfaces as an error
575                // without the event.
576                let serdes_ctx = SerdesContext::new(wire_id, self.ctx.execution_arn());
577                let payload = self.ctx.checkpoint_result_payload(positional_id);
578                let value = replay_success(
579                    &self.ctx,
580                    self.name.as_deref(),
581                    &self.serdes,
582                    payload,
583                    serdes_ctx,
584                )
585                .await?;
586                self.emit_replayed(resumed, view.attempt);
587                Ok(StepPrelude::Done(Ok(value)))
588            }
589            CheckpointStatus::Failed => {
590                self.emit_replayed(resumed, view.attempt);
591                let wire = self
592                    .ctx
593                    .checkpoint_wire_error(positional_id)
594                    .unwrap_or_default();
595                Ok(StepPrelude::Done(Err(replay_failure(wire, wire_id))))
596            }
597            CheckpointStatus::Started => {
598                // AtMostOncePerRetry: the previous attempt was interrupted
599                // before recording an outcome. Do NOT re-execute; treat as
600                // a step-interrupted failure and consult the retry
601                // strategy to decide FAIL or RETRY.
602                let attempt = self.ctx.get_attempt(&self.op_id).saturating_add(1);
603                handle_failure(
604                    &self.ctx,
605                    wire_id,
606                    self.name.as_deref(),
607                    self.sub_type.as_str(),
608                    self.plugin_name.as_deref(),
609                    self.retry_strategy.as_ref(),
610                    interrupted_failure(),
611                    attempt,
612                )
613                .await?;
614                Ok(StepPrelude::Wait)
615            }
616            CheckpointStatus::Unknown(ref raw) => {
617                // Unreachable in production: `checkpoint_view_validated`
618                // already failed the execution (issue #45). Kept as a typed
619                // arm so a future bypass cannot fall through to live
620                // execution and re-run a possibly-terminal step.
621                Err(self.ctx.unrecognized_status_error(wire_id, raw))
622            }
623            // PENDING: the retry timer hasn't fired yet, so wait for it.
624            // `cycle` starts a new attempt for the other statuses of this
625            // arm, so only PENDING reaches it.
626            CheckpointStatus::Pending
627            | CheckpointStatus::Ready
628            | CheckpointStatus::Cancelled
629            | CheckpointStatus::TimedOut
630            | CheckpointStatus::Stopped => Ok(StepPrelude::Wait),
631        }
632    }
633
634    /// What the start step does with this step's record (see
635    /// [`step_cycle`]).
636    fn cycle(&self, view: &crate::engine::CheckpointStatusView) -> crate::start::Cycle {
637        step_cycle(self.semantics, view)
638    }
639
640    /// Fires `on_operation_start` of this step with the status `Started`.
641    fn operation_started(&self, plugins: &crate::plugin::InvocationPlugins, is_replay: bool) {
642        let info = step_info(
643            &self.ctx,
644            self.op_id.wire(),
645            self.sub_type.as_str(),
646            self.name.as_deref(),
647            self.plugin_name.as_deref(),
648        )
649        .with_status(crate::OperationStatus::Started)
650        .with_replay(is_replay);
651        crate::plugin::operation_started(plugins, &info);
652    }
653
654    /// Fires `on_attempt_start` of the live attempt `attempt`, after
655    /// `on_operation_start` fired: in the start step for an at-least-once
656    /// attempt, or after the START response for an at-most-once one.
657    ///
658    /// It returns the attempt's info, which the runner passes to the
659    /// user-function hooks and wraps. Without plugins nothing reads it.
660    fn attempt_started(&self, attempt: u32) -> std::sync::Arc<crate::plugin::OperationInfo> {
661        let info = step_info(
662            &self.ctx,
663            self.op_id.wire(),
664            self.sub_type.as_str(),
665            self.name.as_deref(),
666            self.plugin_name.as_deref(),
667        );
668        match self.ctx.plugins() {
669            Some(plugins) => crate::plugin::attempt_start_only(plugins, info, attempt),
670            None => std::sync::Arc::new(info),
671        }
672    }
673
674    /// Awaits the `START` write an at-least-once attempt submitted before
675    /// its body ran, once the body has returned.
676    ///
677    /// A failed write fails the invocation, and no outcome is written. No
678    /// terminal `FAIL` is needed: at-least-once semantics allow the body to
679    /// run again in the next invocation.
680    async fn await_start(&self, start: Option<SubmittedWrite>) -> Result<(), OperationError> {
681        if let Some(start) = start
682            && let Err(err) = start.await
683        {
684            return self.ctx.checkpoint_failure_unrecoverable(err).await;
685        }
686        Ok(())
687    }
688
689    /// Settles one attempt: checkpoints success or consults the retry
690    /// strategy on failure.
691    async fn settle(
692        &self,
693        attempt: u32,
694        ran: crate::plugin::Ran<O>,
695    ) -> Result<StepSettled<O>, OperationError> {
696        let wire_id = self.op_id.wire();
697        match ran.into_parts() {
698            Ok(value) => handle_success(
699                &self.ctx,
700                wire_id,
701                self.name.as_deref(),
702                self.sub_type.as_str(),
703                self.plugin_name.as_deref(),
704                &self.serdes,
705                value,
706                attempt,
707            )
708            .await
709            .map(StepSettled::Done),
710            Err((err, wire)) => {
711                handle_failure(
712                    &self.ctx,
713                    wire_id,
714                    self.name.as_deref(),
715                    self.sub_type.as_str(),
716                    self.plugin_name.as_deref(),
717                    self.retry_strategy.as_ref(),
718                    (StepError::execution_failed(err), wire),
719                    attempt,
720                )
721                .await?;
722                Ok(StepSettled::Retry)
723            }
724        }
725    }
726}
727
728// Free functions for post-closure operations
729
730/// What the start step does with a step's record under `semantics`: a
731/// `READY`, `CANCELLED`, `TIMED_OUT` or `STOPPED` record begins a new
732/// attempt, which is a replay from the second attempt on. A `STARTED`
733/// record under `AtLeastOncePerRetry` re-enters its attempt, and every
734/// other record queues no `START`.
735pub(crate) fn step_cycle(
736    semantics: StepSemantics,
737    view: &crate::engine::CheckpointStatusView,
738) -> crate::start::Cycle {
739    use crate::start::Cycle;
740    match view.status {
741        CheckpointStatus::Ready
742        | CheckpointStatus::Cancelled
743        | CheckpointStatus::TimedOut
744        | CheckpointStatus::Stopped => Cycle::Start {
745            is_replay: view.attempt > 0,
746        },
747        CheckpointStatus::Started if semantics == StepSemantics::AtLeastOncePerRetry => {
748            Cycle::Resume
749        }
750        _ => Cycle::Record,
751    }
752}
753
754/// The plugin info of the step `wire_id`, created in `ctx`. A
755/// `plugin_name` replaces the name the info reports.
756fn step_info(
757    ctx: &DurableContext,
758    wire_id: &str,
759    sub_type: &str,
760    name: Option<&str>,
761    plugin_name: Option<&str>,
762) -> crate::plugin::OperationInfo {
763    let info = crate::plugin::OperationInfo::for_operation(
764        ctx,
765        wire_id,
766        crate::OperationKind::Step,
767        sub_type,
768        name,
769    );
770    match plugin_name {
771        Some(plugin_name) => info.with_name(Some(plugin_name.to_owned())),
772        None => info,
773    }
774}
775
776/// Handles a successful step execution: serialize, checkpoint, return.
777///
778/// The value round-trips through the configured wire format: ownership
779/// transfers to `serialize`, and the returned value is what `deserialize`
780/// produced from the stored wire string (round-trip parity).
781#[expect(clippy::too_many_arguments)] // reason: the plugin name joins the inputs the end hooks need
782async fn handle_success<O, S: Serdes<O>>(
783    ctx: &DurableContext,
784    wire_id: &str,
785    name: Option<&str>,
786    sub_type: &str,
787    plugin_name: Option<&str>,
788    serdes: &S,
789    value: O,
790    attempt: u32,
791) -> Result<O, OperationError> {
792    let serdes_ctx = SerdesContext::new(wire_id, ctx.execution_arn());
793
794    // Serialize the result (ownership transfers to the serdes).
795    //
796    // A serialization failure is a LOCAL, deterministic, user-facing
797    // failure, so it stays catchable, but the terminal FAIL is persisted
798    // FIRST (issue #43). With the failure recorded, replay yields the
799    // recorded FAIL instead of re-running the body: the body executes
800    // exactly once, and a handler that catches the error branches on a
801    // decision replay reproduces.
802    let serialized = match serialize_value(ctx, name, serdes, value, serdes_ctx.clone()).await {
803        Ok(serialized) => serialized,
804        Err(op_err) => {
805            // The fixed serialization wire type is the replay
806            // discriminator: `replay_failure` matches it to reconstruct
807            // `StepErrorKind::SerializationFailed`, so a handler that
808            // branches on the kind takes the same path live and replayed.
809            let wire = crate::error::serialization_failure_wire(&op_err);
810            let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
811            if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
812                return ctx.checkpoint_failure_unrecoverable(client_err).await;
813            }
814            return Err(op_err
815                .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
816                .with_wire(wire));
817        }
818    };
819
820    // Checkpoint SUCCEED with payload.
821    let update = build_succeed_update(wire_id, name, sub_type, ctx.parent_wire_id(), &serialized);
822    if let Err(err) = ctx.checkpoint_updates(vec![update]).await {
823        return ctx.checkpoint_failure_unrecoverable(err).await;
824    }
825
826    if let Some(plugins) = ctx.plugins() {
827        crate::plugin::attempt_ended(
828            plugins,
829            step_info(ctx, wire_id, sub_type, name, plugin_name)
830                .with_status(crate::OperationStatus::Succeeded)
831                .with_result(Some(serialized.clone())),
832            crate::plugin::AttemptEnd {
833                attempt,
834                outcome: crate::plugin::AttemptOutcome::Succeeded,
835                error: None,
836                ends_operation: true,
837            },
838        );
839    }
840
841    // Return deserialized from the serialized form (round-trip parity).
842    deserialize_result(ctx, name, serdes, serialized, serdes_ctx).await
843}
844
845/// Handles a failed step: consult retry strategy, checkpoint accordingly.
846///
847/// `step_err` is `StepError::execution_failed` of the body's error when
848/// the body failed, and `StepError::interrupted()` when an
849/// `AtMostOncePerRetry` attempt started but recorded no outcome. The retry
850/// strategy sees `step_err`, and the step returns it when the strategy
851/// stops.
852///
853/// Returns `Ok(())` once a `RETRY` is recorded: the caller then waits for
854/// the retry timer. Returns the step error once a `FAIL` is recorded.
855#[expect(clippy::too_many_arguments)] // reason: the subtype joins the inputs every outcome update needs
856async fn handle_failure(
857    ctx: &DurableContext,
858    wire_id: &str,
859    name: Option<&str>,
860    sub_type: &str,
861    plugin_name: Option<&str>,
862    retry_strategy: Option<&RetryStrategy>,
863    (step_err, wire): (StepError, crate::error::WireError),
864    attempt: u32,
865) -> Result<(), OperationError> {
866    // `wire` is the failure record derived once for the error: by the runner
867    // for a live attempt, by `interrupted_failure` for an interrupted one.
868    // The RETRY or FAIL record and the attempt-end hooks all use it.
869    // A plugin sees no hook for an interrupted attempt, as JS: no hook saw
870    // that attempt start in this invocation.
871    let plugins = ctx
872        .plugins()
873        .filter(|_| !matches!(step_err.kind(), StepErrorKind::Interrupted));
874    // Fires the attempt-end hooks once the outcome write returned `Ok`.
875    let attempt_ended = |status: crate::OperationStatus, ends_operation: bool| {
876        if let Some(plugins) = plugins {
877            crate::plugin::attempt_ended(
878                plugins,
879                step_info(ctx, wire_id, sub_type, name, plugin_name)
880                    .with_status(status)
881                    .with_error(Some(wire.clone())),
882                crate::plugin::AttemptEnd {
883                    attempt,
884                    outcome: crate::plugin::AttemptOutcome::Failed,
885                    error: Some(wire.clone()),
886                    ends_operation,
887                },
888            );
889        }
890    };
891
892    // Consult the retry strategy. The strategy sees the live escaping
893    // error through the step error's `source()`.
894    let decision = if let Some(strategy) = retry_strategy {
895        strategy(&step_err, attempt)
896    } else {
897        default_retry_strategy()(&step_err, attempt)
898    };
899
900    match decision {
901        RetryDecision::Retry { delay } => {
902            // Checkpoint RETRY with delay. Round fractional delays UP so the
903            // retry never fires earlier than requested (1.9s -> 2s), enforce
904            // a 1-second minimum, and clamp to i32::MAX.
905            let whole_secs = delay
906                .as_secs()
907                .saturating_add(u64::from(delay.subsec_nanos() > 0));
908            let delay_secs = i32::try_from(whole_secs.max(1)).unwrap_or(i32::MAX);
909            let update = build_retry_update(
910                wire_id,
911                name,
912                sub_type,
913                ctx.parent_wire_id(),
914                &wire,
915                delay_secs,
916            );
917            if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
918                return ctx.checkpoint_failure_unrecoverable(client_err).await;
919            }
920            attempt_ended(crate::OperationStatus::Pending, false);
921            Ok(())
922        }
923        RetryDecision::Stop => {
924            // Checkpoint FAIL (permanent).
925            let update = build_fail_update(wire_id, name, sub_type, ctx.parent_wire_id(), &wire);
926            if let Err(client_err) = ctx.checkpoint_updates(vec![update]).await {
927                return ctx.checkpoint_failure_unrecoverable(client_err).await;
928            }
929            attempt_ended(crate::OperationStatus::Failed, true);
930
931            // The step returns the error the strategy saw, as the JS SDK
932            // does: the kind stays the one the strategy saw, and the
933            // escaping error stays reachable through `source()`.
934            Err(
935                OperationError::from_kind(OperationErrorKind::Step(step_err))
936                    .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
937                    .with_wire(wire),
938            )
939        }
940    }
941}
942
943/// The error and wire of an interrupted attempt. No body ran, so this
944/// path derives its own wire, with the type the JS SDK records.
945fn interrupted_failure() -> (StepError, crate::error::WireError) {
946    let message: BoxError = STEP_INTERRUPTED_MESSAGE.into();
947    let wire = crate::error::wire_error_for(&*message, STEP_INTERRUPTED_ERROR_TYPE);
948    (StepError::interrupted(), wire)
949}
950
951// Update builders
952
953fn build_start_update(
954    wire_id: &str,
955    name: Option<&str>,
956    sub_type: &str,
957    parent_wire_id: Option<&str>,
958) -> OperationUpdate {
959    let mut builder = OperationUpdate::builder()
960        .id(wire_id)
961        .r#type(OperationType::Step)
962        .sub_type(sub_type)
963        .action(OperationAction::Start);
964
965    if let Some(n) = name {
966        builder = builder.name(n);
967    }
968
969    if let Some(parent) = parent_wire_id {
970        builder = builder.parent_id(parent);
971    }
972
973    // build() is infallible here: all required fields (id, type, action) set.
974    #[expect(clippy::expect_used)] // reason: all required fields are set above
975    builder
976        .build()
977        .expect("all required OperationUpdate fields set")
978}
979
980fn build_succeed_update(
981    wire_id: &str,
982    name: Option<&str>,
983    sub_type: &str,
984    parent_wire_id: Option<&str>,
985    payload: &str,
986) -> OperationUpdate {
987    let mut builder = OperationUpdate::builder()
988        .id(wire_id)
989        .r#type(OperationType::Step)
990        .sub_type(sub_type)
991        .action(OperationAction::Succeed)
992        .payload(payload);
993
994    if let Some(n) = name {
995        builder = builder.name(n);
996    }
997
998    if let Some(parent) = parent_wire_id {
999        builder = builder.parent_id(parent);
1000    }
1001
1002    #[expect(clippy::expect_used)] // reason: all required fields are set above
1003    builder
1004        .build()
1005        .expect("all required OperationUpdate fields set")
1006}
1007
1008fn build_retry_update(
1009    wire_id: &str,
1010    name: Option<&str>,
1011    sub_type: &str,
1012    parent_wire_id: Option<&str>,
1013    error: &crate::error::WireError,
1014    delay_secs: i32,
1015) -> OperationUpdate {
1016    let mut builder = OperationUpdate::builder()
1017        .id(wire_id)
1018        .r#type(OperationType::Step)
1019        .sub_type(sub_type)
1020        .action(OperationAction::Retry)
1021        .error(error.to_error_object())
1022        .step_options(
1023            aws_sdk_lambda::types::StepOptions::builder()
1024                .next_attempt_delay_seconds(delay_secs)
1025                .build(),
1026        );
1027
1028    if let Some(n) = name {
1029        builder = builder.name(n);
1030    }
1031
1032    if let Some(parent) = parent_wire_id {
1033        builder = builder.parent_id(parent);
1034    }
1035
1036    #[expect(clippy::expect_used)] // reason: all required fields are set above
1037    builder
1038        .build()
1039        .expect("all required OperationUpdate fields set")
1040}
1041
1042fn build_fail_update(
1043    wire_id: &str,
1044    name: Option<&str>,
1045    sub_type: &str,
1046    parent_wire_id: Option<&str>,
1047    error: &crate::error::WireError,
1048) -> OperationUpdate {
1049    let mut builder = OperationUpdate::builder()
1050        .id(wire_id)
1051        .r#type(OperationType::Step)
1052        .sub_type(sub_type)
1053        .action(OperationAction::Fail)
1054        .error(error.to_error_object());
1055
1056    if let Some(n) = name {
1057        builder = builder.name(n);
1058    }
1059
1060    if let Some(parent) = parent_wire_id {
1061        builder = builder.parent_id(parent);
1062    }
1063
1064    #[expect(clippy::expect_used)] // reason: all required fields are set above
1065    builder
1066        .build()
1067        .expect("all required OperationUpdate fields set")
1068}
1069
1070// Serialization helpers
1071
1072/// Serializes a step value through the configured serdes.
1073///
1074/// The serdes decides where its work runs (inline for `JsonSerdes`, one
1075/// blocking task for `FileSystemSerdes`); the SDK awaits the returned
1076/// future directly. Taking `value` by ownership keeps the future `Send`
1077/// without requiring `O: Sync`.
1078///
1079/// A marked error ends the invocation
1080/// ([`DurableContext::route_serdes_error`]); any other error becomes
1081/// `SerializationFailed`.
1082async fn serialize_value<O, S: Serdes<O>>(
1083    ctx: &DurableContext,
1084    name: Option<&str>,
1085    serdes: &S,
1086    value: O,
1087    serdes_ctx: SerdesContext,
1088) -> Result<String, OperationError> {
1089    let operation_id = serdes_ctx.operation_id().to_owned();
1090    match ctx.hold_activity(serdes.serialize(value, serdes_ctx)).await {
1091        Ok(serialized) => Ok(serialized),
1092        Err(err) => Err(step_serialization_error(
1093            ctx.route_serdes_error("serialize", &operation_id, name, err)
1094                .await,
1095        )),
1096    }
1097}
1098
1099/// Deserializes a step wire string through the configured serdes.
1100///
1101/// A marked error ends the invocation
1102/// ([`DurableContext::route_serdes_error`]); any other error becomes
1103/// `SerializationFailed`.
1104async fn deserialize_result<O, S: Serdes<O>>(
1105    ctx: &DurableContext,
1106    name: Option<&str>,
1107    serdes: &S,
1108    serialized: String,
1109    serdes_ctx: SerdesContext,
1110) -> Result<O, OperationError> {
1111    let operation_id = serdes_ctx.operation_id().to_owned();
1112    match ctx
1113        .hold_activity(serdes.deserialize(serialized, serdes_ctx))
1114        .await
1115    {
1116        Ok(value) => Ok(value),
1117        Err(err) => Err(step_serialization_error(
1118            ctx.route_serdes_error("deserialize", &operation_id, name, err)
1119                .await,
1120        )),
1121    }
1122}
1123
1124/// Wraps a serdes error as a step `SerializationFailed` operation error,
1125/// carrying the error itself as the source.
1126fn step_serialization_error(e: BoxError) -> OperationError {
1127    OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1128        StepErrorKind::SerializationFailed,
1129        Some(e),
1130    )))
1131}
1132
1133async fn replay_success<O, S: Serdes<O>>(
1134    ctx: &DurableContext,
1135    name: Option<&str>,
1136    serdes: &S,
1137    result: Option<String>,
1138    serdes_ctx: SerdesContext,
1139) -> Result<O, OperationError> {
1140    let payload = result.unwrap_or_else(|| "null".to_owned());
1141    deserialize_result(ctx, name, serdes, payload, serdes_ctx).await
1142}
1143
1144/// Rebuilds a step error from the failure fields of a replayed record.
1145///
1146/// The recorded wire fields travel on the synthetic source (a
1147/// [`crate::error::ReplayedFailure`]) rather than being folded into a
1148/// message, so `kind()` remains meaningful after a replay and the
1149/// recorded `error_type` stays programmatically recoverable through a
1150/// `source()` downcast.
1151///
1152/// A record whose `error_type` is the serialization discriminator
1153/// ([`crate::error::SERIALIZATION_FAILED_ERROR_TYPE`]) reconstructs
1154/// `StepErrorKind::SerializationFailed`, the kind the live path yielded
1155/// after persisting that record (issue #43). A record whose `error_type`
1156/// is [`STEP_INTERRUPTED_ERROR_TYPE`] reconstructs
1157/// `StepErrorKind::Interrupted`, the kind the live path yielded after the
1158/// retry strategy stopped an interrupted `AtMostOncePerRetry` attempt.
1159/// Every other record is an
1160/// ordinary terminal step failure, which the live path yields as
1161/// `StepErrorKind::ExecutionFailed` after its retry strategy stopped, so
1162/// replay reconstructs the SAME kind: a handler that branches on the
1163/// kind takes the same path live and replayed.
1164fn replay_failure(wire: crate::error::WireError, wire_id: &str) -> OperationError {
1165    let kind = match wire.error_type() {
1166        Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE) => StepErrorKind::SerializationFailed,
1167        Some(STEP_INTERRUPTED_ERROR_TYPE) => StepErrorKind::Interrupted,
1168        _ => StepErrorKind::ExecutionFailed,
1169    };
1170    OperationError::from_kind(OperationErrorKind::Step(StepError::new(
1171        kind,
1172        Some(crate::error::ReplayedFailure::source_from(wire.clone())),
1173    )))
1174    .with_operation(wire_id, CheckpointStatus::Failed.wire_str())
1175    .with_wire(wire)
1176}
1177
1178/// The wire `ErrorType` recorded for a step failure whose escaping error
1179/// carries no structured identity.
1180///
1181/// A step body returns `Result<O, BoxError>`, so the concrete error type
1182/// is erased *by the caller* before the SDK ever sees it: `?` boxes the
1183/// error at the user's call site, and Rust offers no runtime name for a
1184/// `dyn Error`. Rather than guessing one from a `Debug` rendering, the
1185/// SDK records this explicit generic name. An error that *does* carry
1186/// structured identity: an [`OperationError`], or a
1187/// [`crate::error::ReplayedFailure`]: records its registry name or its
1188/// original recorded type instead (see [`crate::error::wire_error_for`]).
1189const STEP_FALLBACK_ERROR_TYPE: &str = "Error";
1190
1191/// The wire `ErrorType` recorded for an `AtMostOncePerRetry` attempt that
1192/// started but recorded no outcome. It is the name the JS SDK records.
1193const STEP_INTERRUPTED_ERROR_TYPE: &str = "StepInterruptedError";
1194
1195#[cfg(test)]
1196#[expect(clippy::panic)]
1197// reason: test assertions with descriptive messages
1198// Tests deliberately spawn foreign (unblessed) tasks to exercise runtime
1199// behavior; production spawning is confined to the src/task.rs gateway.
1200#[expect(clippy::disallowed_methods)]
1201mod tests {
1202    use super::*;
1203    use crate::context::DurableContext;
1204    use crate::engine::CheckpointLog;
1205    use crate::plugin::recorder::UfHooks;
1206    use std::sync::Arc;
1207
1208    // Replay tests
1209
1210    #[tokio::test]
1211    async fn replay_success_deserializes_json() {
1212        let payload = "42".to_owned();
1213        let ctx = SerdesContext::new("op-1", "arn:test");
1214        let (dctx, _client) = context_over(Vec::new());
1215        let result: Result<i32, OperationError> =
1216            replay_success(&dctx, None, &crate::serdes::JsonSerdes, Some(payload), ctx).await;
1217        assert_eq!(result.unwrap(), 42);
1218    }
1219
1220    #[tokio::test]
1221    async fn replay_success_null_returns_unit() {
1222        let ctx = SerdesContext::new("op-1", "arn:test");
1223        let (dctx, _client) = context_over(Vec::new());
1224        let result: Result<(), OperationError> =
1225            replay_success(&dctx, None, &crate::serdes::JsonSerdes, None, ctx).await;
1226        assert!(result.is_ok());
1227    }
1228
1229    #[test]
1230    fn replay_failure_keeps_kind_and_wire_fields_apart() {
1231        let wire = crate::error::WireError::new(Some("TypeError"), Some("bad input"));
1232        let err = replay_failure(wire, "wire-1");
1233        // kind() is meaningful after a replay: an ordinary terminal step
1234        // failure replays as ExecutionFailed, the kind the live path
1235        // yielded.
1236        let OperationErrorKind::Step(step_err) = err.kind() else {
1237            unreachable!("replay builds a step error");
1238        };
1239        let StepErrorKind::ExecutionFailed = step_err.kind() else {
1240            unreachable!("ordinary failures replay as ExecutionFailed");
1241        };
1242        // The type is NOT folded into the message: the frame stays clean...
1243        assert_eq!(err.to_string(), "operation error: step");
1244        // ...and the recorded fields are programmatically recoverable.
1245        let source = std::error::Error::source(step_err).expect("synthetic source");
1246        let replayed = source
1247            .downcast_ref::<crate::error::ReplayedFailure>()
1248            .expect("replay source is a ReplayedFailure");
1249        assert_eq!(replayed.error_type(), Some("TypeError"));
1250        assert_eq!(replayed.error_message(), Some("bad input"));
1251        // The wire record, operation id, and status are reachable.
1252        assert_eq!(err.operation_id(), Some("wire-1"));
1253        assert_eq!(err.status(), Some("FAILED"));
1254        assert_eq!(
1255            err.wire().and_then(crate::error::WireError::error_type),
1256            Some("TypeError")
1257        );
1258        // The chain renders the recorded message via the alternate form:
1259        // without folding the type into the text.
1260        let chain = format!("{err:#}");
1261        assert!(chain.contains("bad input"), "got: {chain}");
1262        assert!(
1263            !chain.contains("TypeError"),
1264            "type must not fold into text: {chain}"
1265        );
1266    }
1267
1268    #[test]
1269    fn replay_failure_empty_record_displays_unknown() {
1270        let err = replay_failure(crate::error::WireError::default(), "wire-1");
1271        let chain = format!("{err:#}");
1272        assert!(chain.contains("unknown error"), "got: {chain}");
1273    }
1274
1275    #[test]
1276    fn replay_failure_serialization_discriminator_keeps_its_kind() {
1277        let wire = crate::error::WireError::new(
1278            Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE),
1279            Some("bad payload"),
1280        );
1281        let err = replay_failure(wire, "wire-1");
1282        let OperationErrorKind::Step(step_err) = err.kind() else {
1283            unreachable!("replay builds a step error");
1284        };
1285        assert!(matches!(
1286            step_err.kind(),
1287            StepErrorKind::SerializationFailed
1288        ));
1289    }
1290
1291    // Serialization tests
1292
1293    #[tokio::test]
1294    async fn serialize_deserialize_round_trip() {
1295        let ctx = SerdesContext::new("op-1", "arn:test");
1296        let (dctx, _client) = context_over(Vec::new());
1297        let serialized = serialize_value(
1298            &dctx,
1299            None,
1300            &crate::serdes::JsonSerdes,
1301            "hello".to_owned(),
1302            ctx.clone(),
1303        )
1304        .await
1305        .unwrap();
1306        let deserialized: String =
1307            deserialize_result(&dctx, None, &crate::serdes::JsonSerdes, serialized, ctx)
1308                .await
1309                .unwrap();
1310        assert_eq!(deserialized, "hello");
1311    }
1312
1313    #[tokio::test]
1314    async fn serialize_with_custom_serdes_uppercases() {
1315        struct Upper;
1316        impl Serdes<String> for Upper {
1317            // reason: exercises the async-fn impl form user code writes
1318            #[expect(clippy::unused_async_trait_impl)]
1319            async fn serialize(
1320                &self,
1321                value: String,
1322                _context: SerdesContext,
1323            ) -> Result<String, BoxError> {
1324                Ok(serde_json::to_string(&value)?.to_uppercase())
1325            }
1326            // reason: exercises the async-fn impl form user code writes
1327            #[expect(clippy::unused_async_trait_impl)]
1328            async fn deserialize(
1329                &self,
1330                wire: String,
1331                _context: SerdesContext,
1332            ) -> Result<String, BoxError> {
1333                Ok(serde_json::from_str(&wire)?)
1334            }
1335        }
1336        let ctx = SerdesContext::new("op-1", "arn:test");
1337        let (dctx, _client) = context_over(Vec::new());
1338        let result = serialize_value(&dctx, None, &Upper, "hello".to_owned(), ctx)
1339            .await
1340            .unwrap();
1341        assert_eq!(result, "\"HELLO\"");
1342    }
1343
1344    // Default retry strategy tests
1345
1346    #[test]
1347    fn default_retry_stops_at_max_attempts() {
1348        let strategy = default_retry_strategy();
1349        let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1350        // Attempt 6 should stop (max_attempts = 6).
1351        let decision = strategy(&err, 6);
1352        assert_eq!(decision, RetryDecision::Stop);
1353    }
1354
1355    #[test]
1356    fn default_retry_retries_below_max() {
1357        let strategy = default_retry_strategy();
1358        let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1359        let decision = strategy(&err, 1);
1360        match decision {
1361            RetryDecision::Retry { delay } => {
1362                // With full jitter on 5s base, delay should be 1-5s.
1363                assert!(delay.as_secs() >= 1);
1364                assert!(delay.as_secs() <= 5);
1365            }
1366            RetryDecision::Stop => panic!("expected retry for attempt 1"),
1367        }
1368    }
1369
1370    #[test]
1371    fn default_retry_delay_grows_with_attempts() {
1372        // Attempt 5: base = 5 * 2^4 = 80, capped at 60.
1373        // With full jitter: [0, 60] → rounded: [1, 60].
1374        let strategy = default_retry_strategy();
1375        let err = StepError::new(StepErrorKind::ExecutionFailed, Some("fail".into()));
1376        let decision = strategy(&err, 5);
1377        match decision {
1378            RetryDecision::Retry { delay } => {
1379                // Max possible is 60s.
1380                assert!(delay.as_secs() <= 60);
1381                assert!(delay.as_secs() >= 1);
1382            }
1383            RetryDecision::Stop => panic!("expected retry for attempt 5"),
1384        }
1385    }
1386
1387    // Live execution tests (with mock client)
1388
1389    #[tokio::test]
1390    async fn step_live_success_checkpoints_and_returns() {
1391        use crate::client::InMemoryExecutionClient;
1392
1393        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1394        let log = Arc::new(CheckpointLog::empty());
1395        let ctx = DurableContext::new_root_with_client(
1396            "arn:test".to_owned(),
1397            lambda_runtime::Context::default(),
1398            log,
1399            client,
1400            "token0".to_owned(),
1401        );
1402        let op_id = ctx.mint_id();
1403        let closure = |_: StepContext| async { Ok::<i32, BoxError>(42) };
1404
1405        let exec = StepExecution {
1406            ctx,
1407            op_id,
1408            name: Some("test-step".to_owned()),
1409            retry_strategy: None,
1410            serdes: crate::serdes::JsonSerdes,
1411            semantics: StepSemantics::default(),
1412            sub_type: OperationSubType::STEP,
1413            plugin_name: None,
1414            closure,
1415            _marker: std::marker::PhantomData,
1416        };
1417
1418        let result = exec.execute().await;
1419        assert_eq!(result.unwrap(), 42);
1420    }
1421
1422    #[tokio::test]
1423    async fn per_operation_serdes_serializes_step_result() {
1424        use crate::client::InMemoryExecutionClient;
1425
1426        // A per-operation serdes that uppercases on serialize and lowercases
1427        // on deserialize. The execution-wide serdes slot was removed with
1428        // the generic trait; per-operation configuration is the only path.
1429        struct Upper;
1430        impl Serdes<String> for Upper {
1431            // reason: exercises the async-fn impl form user code writes
1432            #[expect(clippy::unused_async_trait_impl)]
1433            async fn serialize(
1434                &self,
1435                value: String,
1436                _context: SerdesContext,
1437            ) -> Result<String, BoxError> {
1438                Ok(serde_json::to_string(&value)?.to_uppercase())
1439            }
1440            // reason: exercises the async-fn impl form user code writes
1441            #[expect(clippy::unused_async_trait_impl)]
1442            async fn deserialize(
1443                &self,
1444                wire: String,
1445                _context: SerdesContext,
1446            ) -> Result<String, BoxError> {
1447                Ok(serde_json::from_str(&wire.to_lowercase())?)
1448            }
1449        }
1450
1451        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1452        let log = Arc::new(CheckpointLog::empty());
1453        let ctx = DurableContext::new_root_with_client_and_defaults(
1454            "arn:test".to_owned(),
1455            lambda_runtime::Context::default(),
1456            log,
1457            client.clone(),
1458            "token0".to_owned(),
1459            Duration::ZERO,
1460            0,
1461            None,
1462        );
1463        let op_id = ctx.mint_id();
1464        let closure = |_: StepContext| async { Ok::<String, BoxError>("hello".to_owned()) };
1465
1466        let exec = StepExecution {
1467            ctx,
1468            op_id,
1469            name: Some("greet".to_owned()),
1470            retry_strategy: None,
1471            serdes: Upper,
1472            semantics: StepSemantics::default(),
1473            sub_type: OperationSubType::STEP,
1474            plugin_name: None,
1475            closure,
1476            _marker: std::marker::PhantomData,
1477        };
1478
1479        // The step returns the round-tripped value.
1480        let result = exec.execute().await;
1481        assert_eq!(result.unwrap(), "hello");
1482
1483        // The Succeed checkpoint payload was serialized by the configured
1484        // serdes (uppercased). Without the serdes the payload would be the
1485        // plain "\"hello\"".
1486        let updates = client.recorded_updates();
1487        let succeed = updates
1488            .iter()
1489            .find(|u| matches!(u.action(), OperationAction::Succeed))
1490            .expect("a Succeed update must be recorded");
1491        assert_eq!(
1492            succeed.payload(),
1493            Some("\"HELLO\""),
1494            "the per-operation serdes must serialize the step result"
1495        );
1496    }
1497
1498    #[tokio::test]
1499    async fn step_replay_returns_frozen_result_without_re_executing() {
1500        use crate::engine::{CheckpointLog, CheckpointRecord};
1501        use std::sync::atomic::{AtomicBool, Ordering};
1502
1503        let executed = Arc::new(AtomicBool::new(false));
1504        let executed_clone = Arc::clone(&executed);
1505
1506        // Build a checkpoint log with a succeeded record at position "1".
1507        // The log is keyed by wire ID (hash of positional "1"), matching
1508        // how parse_inline_operations stores records from the backend.
1509        let wire_key = crate::engine::compute_wire_id_public("1");
1510        let record = CheckpointRecord {
1511            id: wire_key.clone(),
1512            status: CheckpointStatus::Succeeded,
1513            result: Some("99".to_owned()),
1514            error_type: None,
1515            error_message: None,
1516            error_data: None,
1517            stack_trace: None,
1518            attempt: 0,
1519            invoke_result: None,
1520            invoke_error_type: None,
1521            invoke_error_message: None,
1522            invoke_error_data: None,
1523            invoke_stack_trace: None,
1524            replay_children: false,
1525            callback_id: None,
1526            op_type: None,
1527            sub_type: None,
1528            due_at: None,
1529            op_name: None,
1530            parent_id: None,
1531            start_timestamp: None,
1532            end_timestamp: None,
1533        };
1534        let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
1535        let ctx = DurableContext::new_root(
1536            "arn:test".to_owned(),
1537            lambda_runtime::Context::default(),
1538            log,
1539        );
1540        let op_id = ctx.mint_id(); // This mints "1"
1541
1542        let closure = move |_: StepContext| {
1543            executed_clone.store(true, Ordering::SeqCst);
1544            async { Ok::<i32, BoxError>(0) }
1545        };
1546
1547        let exec = StepExecution {
1548            ctx,
1549            op_id,
1550            name: None,
1551            retry_strategy: None,
1552            serdes: crate::serdes::JsonSerdes,
1553            semantics: StepSemantics::default(),
1554            sub_type: OperationSubType::STEP,
1555            plugin_name: None,
1556            closure,
1557            _marker: std::marker::PhantomData,
1558        };
1559
1560        let result = exec.execute().await;
1561        assert_eq!(result.unwrap(), 99);
1562        assert!(
1563            !executed.load(Ordering::SeqCst),
1564            "closure should NOT execute during replay"
1565        );
1566    }
1567
1568    #[tokio::test]
1569    async fn step_replay_failure_returns_error_without_re_executing() {
1570        use crate::engine::{CheckpointLog, CheckpointRecord};
1571        use std::sync::atomic::{AtomicBool, Ordering};
1572
1573        let executed = Arc::new(AtomicBool::new(false));
1574        let executed_clone = Arc::clone(&executed);
1575
1576        let wire_key = crate::engine::compute_wire_id_public("1");
1577        let record = CheckpointRecord {
1578            id: wire_key.clone(),
1579            status: CheckpointStatus::Failed,
1580            result: None,
1581            error_type: Some("CustomError".to_owned()),
1582            error_message: Some("it broke".to_owned()),
1583            error_data: None,
1584            stack_trace: None,
1585            attempt: 0,
1586            invoke_result: None,
1587            invoke_error_type: None,
1588            invoke_error_message: None,
1589            invoke_error_data: None,
1590            invoke_stack_trace: None,
1591            replay_children: false,
1592            callback_id: None,
1593            op_type: None,
1594            sub_type: None,
1595            due_at: None,
1596            op_name: None,
1597            parent_id: None,
1598            start_timestamp: None,
1599            end_timestamp: None,
1600        };
1601        let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
1602        let ctx = DurableContext::new_root(
1603            "arn:test".to_owned(),
1604            lambda_runtime::Context::default(),
1605            log,
1606        );
1607        let op_id = ctx.mint_id();
1608
1609        let closure = move |_: StepContext| {
1610            executed_clone.store(true, Ordering::SeqCst);
1611            async { Ok::<String, BoxError>("should not run".to_owned()) }
1612        };
1613
1614        let exec = StepExecution {
1615            ctx,
1616            op_id,
1617            name: None,
1618            retry_strategy: None,
1619            serdes: crate::serdes::JsonSerdes,
1620            semantics: StepSemantics::default(),
1621            sub_type: OperationSubType::STEP,
1622            plugin_name: None,
1623            closure,
1624            _marker: std::marker::PhantomData,
1625        };
1626
1627        let result = exec.execute().await;
1628        assert!(result.is_err());
1629        let err = result.unwrap_err();
1630        let err_msg = format!("{err:#}");
1631        assert!(err_msg.contains("it broke"), "got: {err_msg}");
1632        // The recorded type is wire data, not display text.
1633        assert_eq!(
1634            err.wire().and_then(crate::error::WireError::error_type),
1635            Some("CustomError")
1636        );
1637        assert!(
1638            !executed.load(Ordering::SeqCst),
1639            "closure should NOT execute during replay"
1640        );
1641    }
1642
1643    #[tokio::test]
1644    async fn step_retry_then_stop_on_exhaustion() {
1645        use crate::client::InMemoryExecutionClient;
1646
1647        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
1648        let log = Arc::new(CheckpointLog::empty());
1649        let ctx = DurableContext::new_root_with_client(
1650            "arn:test".to_owned(),
1651            lambda_runtime::Context::default(),
1652            log,
1653            client,
1654            "token0".to_owned(),
1655        );
1656        let op_id = ctx.mint_id();
1657
1658        // A strategy that stops immediately (max_attempts = 1).
1659        let no_retry: RetryStrategy =
1660            Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
1661
1662        let closure = |_: StepContext| async { Err::<i32, BoxError>("always fails".into()) };
1663
1664        let exec = StepExecution {
1665            ctx,
1666            op_id,
1667            name: None,
1668            retry_strategy: Some(no_retry),
1669            serdes: crate::serdes::JsonSerdes,
1670            semantics: StepSemantics::default(),
1671            sub_type: OperationSubType::STEP,
1672            plugin_name: None,
1673            closure,
1674            _marker: std::marker::PhantomData,
1675        };
1676
1677        let result = exec.execute().await;
1678        assert!(result.is_err());
1679        let err_msg = format!("{:#}", result.unwrap_err());
1680        assert!(
1681            err_msg.contains("retries exhausted") || err_msg.contains("always fails"),
1682            "got: {err_msg}"
1683        );
1684    }
1685
1686    /// A root context on the in-memory client, with `hooks` as the
1687    /// invocation's plugin, and the checkpoint buffer window `window`.
1688    fn context_with_plugin(
1689        hooks: &crate::plugin::test_support::OpHooks,
1690        window: Duration,
1691    ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
1692        let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
1693        let ctx = DurableContext::new_root_with_client_and_defaults(
1694            "arn:test".to_owned(),
1695            lambda_runtime::Context::default(),
1696            Arc::new(CheckpointLog::empty()),
1697            Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
1698            "token0".to_owned(),
1699            window,
1700            0,
1701            Some(hooks.plugins()),
1702        );
1703        (ctx, client)
1704    }
1705
1706    #[tokio::test]
1707    async fn hooks_run_outside_the_token_lock() {
1708        let lock: Arc<std::sync::OnceLock<Arc<tokio::sync::Mutex<String>>>> = Arc::default();
1709        let probe_lock = Arc::clone(&lock);
1710        let hooks = crate::plugin::test_support::OpHooks::probing(move || {
1711            let free = probe_lock
1712                .get()
1713                .is_some_and(|mutex| mutex.try_lock().is_ok());
1714            free.to_string()
1715        });
1716        let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1717        lock.set(ctx.checkpoint_token_mutex()).expect("set once");
1718        let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
1719        assert_eq!(value, 1);
1720        let probes: Vec<String> = hooks.records().into_iter().map(|r| r.probe).collect();
1721        assert_eq!(probes, ["true"; 4]);
1722    }
1723
1724    #[tokio::test]
1725    async fn hooks_run_on_the_operation_task() {
1726        let hooks = crate::plugin::test_support::OpHooks::probing(|| {
1727            format!("{:?}", tokio::task::try_id())
1728        });
1729        let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1730        let body_task = Arc::new(std::sync::Mutex::new(None));
1731        let seen = Arc::clone(&body_task);
1732        let value = ctx
1733            .step(move |_| async move {
1734                *seen.lock().unwrap() = tokio::task::try_id();
1735                Ok(1_u32)
1736            })
1737            .name("a")
1738            .spawn()
1739            .await
1740            .unwrap();
1741        assert_eq!(value, 1);
1742        let body = body_task.lock().unwrap().expect("the body ran on a task");
1743        let records = hooks.records();
1744        let probe_of = |hook: &str| {
1745            records
1746                .iter()
1747                .find(|r| r.hook == hook)
1748                .map(|r| r.probe.clone())
1749                .expect("the hook ran")
1750        };
1751        let expected = format!("{:?}", Some(body));
1752        assert_eq!(probe_of("attempt_end"), expected);
1753        assert_eq!(probe_of("end"), expected);
1754    }
1755
1756    #[tokio::test]
1757    async fn no_plugins_keeps_checkpoint_updates() {
1758        let (ctx, client) = context_over(Vec::new());
1759        assert!(ctx.plugins().is_none());
1760        let value = ctx.step(|_| async { Ok(1_u32) }).name("a").await.unwrap();
1761        assert_eq!(value, 1);
1762        let updates = client.recorded_updates();
1763        let wire_id = crate::engine::compute_wire_id_public("1");
1764        let fields: Vec<_> = updates
1765            .iter()
1766            .map(|u| {
1767                (
1768                    u.id().to_owned(),
1769                    u.r#type().clone(),
1770                    u.sub_type().map(str::to_owned),
1771                    u.name().map(str::to_owned),
1772                    u.action().clone(),
1773                    u.payload().map(str::to_owned),
1774                    u.parent_id().map(str::to_owned),
1775                    u.error().is_some(),
1776                )
1777            })
1778            .collect();
1779        let step = |action, payload: Option<&str>| {
1780            (
1781                wire_id.clone(),
1782                OperationType::Step,
1783                Some(STEP_SUB_TYPE.to_owned()),
1784                Some("a".to_owned()),
1785                action,
1786                payload.map(str::to_owned),
1787                None,
1788                false,
1789            )
1790        };
1791        assert_eq!(
1792            fields,
1793            [
1794                step(OperationAction::Start, None),
1795                step(OperationAction::Succeed, Some("1")),
1796            ]
1797        );
1798    }
1799
1800    /// The SDK's in-memory client returns no record, so the end infos of a
1801    /// failed step carry the error the SDK wrote.
1802    #[tokio::test]
1803    async fn failed_step_end_infos_carry_the_written_error() {
1804        let hooks = crate::plugin::test_support::OpHooks::default();
1805        let (ctx, _client) = context_with_plugin(&hooks, Duration::ZERO);
1806        let err = ctx
1807            .step(|_| async { Err::<u32, BoxError>("boom".into()) })
1808            .name("a")
1809            .retry_strategy(|_err, _attempt| RetryDecision::Stop)
1810            .await
1811            .unwrap_err();
1812        assert!(err.wire().is_some());
1813        let records = hooks.records();
1814        let ends: Vec<_> = records
1815            .iter()
1816            .filter(|r| r.hook == "attempt_end" || r.hook == "end")
1817            .collect();
1818        assert_eq!(ends.len(), 2);
1819        let attempt_end = ends.first().expect("an attempt end");
1820        assert_eq!(attempt_end.error.as_deref(), Some("boom"));
1821        assert_eq!(attempt_end.status, Some(crate::OperationStatus::Failed));
1822    }
1823
1824    /// A root context on the in-memory client over `records`, with the
1825    /// recorder's `p0` and `p1` as the invocation's plugins and no
1826    /// checkpoint buffer window.
1827    fn uf_context(
1828        hooks: &UfHooks,
1829        records: Vec<(String, crate::engine::CheckpointRecord)>,
1830    ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
1831        let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
1832        let ctx = DurableContext::new_root_with_client_and_defaults(
1833            "arn:test".to_owned(),
1834            lambda_runtime::Context::default(),
1835            Arc::new(CheckpointLog::from_records(records)),
1836            Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
1837            "token0".to_owned(),
1838            Duration::ZERO,
1839            0,
1840            Some(hooks.plugins()),
1841        );
1842        (ctx, client)
1843    }
1844
1845    /// Drives `fut` on `ctx`'s invocation driver to its outcome.
1846    async fn drive<F>(ctx: &DurableContext, fut: F) -> crate::driver::InvocationOutcome
1847    where
1848        F: IntoFuture + Send + 'static,
1849        F::IntoFuture: Send + 'static,
1850    {
1851        let signal = Arc::clone(ctx.coordinator());
1852        let ownership = Arc::clone(ctx.task_ownership());
1853        crate::driver::test_support::outcome_of(ctx.clone(), ownership, signal, fut).await
1854    }
1855
1856    /// The first update `client` recorded for `name` with `action`.
1857    fn update_of(
1858        client: &crate::client::InMemoryExecutionClient,
1859        name: &str,
1860        action: &OperationAction,
1861    ) -> Option<OperationUpdate> {
1862        client
1863            .recorded_updates()
1864            .into_iter()
1865            .find(|u| u.name() == Some(name) && u.action() == action)
1866    }
1867
1868    /// `(type, message)` of a wire error.
1869    fn type_and_message(wire: &crate::error::WireError) -> (Option<String>, Option<String>) {
1870        (
1871            wire.error_type().map(str::to_owned),
1872            wire.error_message().map(str::to_owned),
1873        )
1874    }
1875
1876    /// `(type, message)` of the error object of `update`.
1877    fn update_error(update: &OperationUpdate) -> (Option<String>, Option<String>) {
1878        let error = update.error().expect("the update carries an error object");
1879        (
1880            error.error_type().map(str::to_owned),
1881            error.error_message().map(str::to_owned),
1882        )
1883    }
1884
1885    /// The user-function records of one live attempt, in `p0`'s order.
1886    const LIVE_ATTEMPT: [&str; 6] = [
1887        "attempt_start",
1888        "uf_start",
1889        "wrap_enter",
1890        "wrap_exit",
1891        "uf_end",
1892        "attempt_end",
1893    ];
1894
1895    /// Rule 7.8: an interrupted `AtMostOncePerRetry` attempt runs no
1896    /// body and fires no user-function hook; it records a RETRY. The next
1897    /// attempt is a user function.
1898    #[tokio::test]
1899    async fn interrupted_attempt_fires_no_user_function_hook() {
1900        use std::sync::atomic::{AtomicU32, Ordering};
1901        let runs = Arc::new(AtomicU32::new(0));
1902        let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
1903            let runs = Arc::clone(runs);
1904            ctx.step(move |_| async move {
1905                runs.fetch_add(1, Ordering::SeqCst);
1906                Ok(42_u32)
1907            })
1908            .name("s")
1909            .semantics(StepSemantics::AtMostOncePerRetry)
1910            .retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(1)))
1911        };
1912
1913        // Invocation 1: attempt 1 is STARTED.
1914        let hooks = UfHooks::default();
1915        let (ctx, client) = uf_context(
1916            &hooks,
1917            vec![step_record(CheckpointStatus::Started, 0, None)],
1918        );
1919        let outcome = drive(&ctx, step(&ctx, &runs)).await;
1920        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1921        assert_eq!(runs.load(Ordering::SeqCst), 0, "the body did not run");
1922        let retry = recorded_error_for(&client, &OperationAction::Retry);
1923        assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
1924        assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
1925
1926        // Invocation 2: the retry timer fired, so attempt 2 runs.
1927        let hooks = UfHooks::default();
1928        let (ctx, _client) =
1929            uf_context(&hooks, vec![step_record(CheckpointStatus::Ready, 1, None)]);
1930        assert_eq!(step(&ctx, &runs).await.unwrap(), 42);
1931        let mut expected = vec!["start"];
1932        expected.extend(LIVE_ATTEMPT);
1933        expected.push("end");
1934        assert_eq!(hooks.hooks_of("s"), expected);
1935        assert_eq!(hooks.find("s", "uf_start").info_attempt, Some(2));
1936        assert_eq!(hooks.find("s", "uf_end").ok, Some(true));
1937    }
1938
1939    /// Rule 7.8: a step in its retry delay runs no body, builds no node
1940    /// and fires no user-function hook.
1941    #[tokio::test]
1942    async fn retry_wait_fires_no_user_function_hook() {
1943        use std::sync::atomic::{AtomicU32, Ordering};
1944        let runs = Arc::new(AtomicU32::new(0));
1945        let step = |ctx: &DurableContext, runs: &Arc<AtomicU32>| {
1946            let runs = Arc::clone(runs);
1947            ctx.step(move |_| async move {
1948                runs.fetch_add(1, Ordering::SeqCst);
1949                Err::<u32, BoxError>("transient".into())
1950            })
1951            .name("s")
1952            .retry_strategy(|_err, _attempt| RetryDecision::retry(Duration::from_secs(10)))
1953        };
1954
1955        // Invocation 1: attempt 1 fails and schedules a 10 s retry.
1956        let hooks = UfHooks::default();
1957        let (ctx, client) = uf_context(&hooks, Vec::new());
1958        let outcome = drive(&ctx, step(&ctx, &runs)).await;
1959        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1960        assert_eq!(runs.load(Ordering::SeqCst), 1);
1961        let mut expected = vec!["start"];
1962        expected.extend(LIVE_ATTEMPT);
1963        assert_eq!(hooks.hooks_of("s"), expected);
1964        assert_eq!(hooks.find("s", "uf_end").ok, Some(false));
1965        assert!(update_of(&client, "s", &OperationAction::Retry).is_some());
1966
1967        // Invocation 2 starts within the delay: the record is PENDING.
1968        let hooks = UfHooks::default();
1969        let (wire_key, mut record) = step_record(CheckpointStatus::Pending, 1, None);
1970        record.due_at = Some(std::time::SystemTime::now() + Duration::from_secs(10));
1971        let (ctx, client) = uf_context(&hooks, vec![(wire_key, record)]);
1972        let outcome = drive(&ctx, step(&ctx, &runs)).await;
1973        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
1974        assert_eq!(
1975            runs.load(Ordering::SeqCst),
1976            1,
1977            "no body ran in invocation 2"
1978        );
1979        assert!(!hooks.any_user_function_record(), "{:?}", hooks.records());
1980        assert!(
1981            client.recorded_updates().is_empty(),
1982            "the step stays PENDING: {:?}",
1983            client.recorded_updates()
1984        );
1985    }
1986
1987    /// The end hook runs before the outcome is checkpointed: the START is
1988    /// recorded, the SUCCEED is not.
1989    #[tokio::test]
1990    async fn end_hook_before_outcome_record() {
1991        let client_cell: Arc<std::sync::OnceLock<Arc<crate::client::InMemoryExecutionClient>>> =
1992            Arc::default();
1993        let probe_client = Arc::clone(&client_cell);
1994        let hooks = UfHooks::probing(move || {
1995            let actions: Vec<String> = probe_client
1996                .get()
1997                .map(|client| client.recorded_updates())
1998                .unwrap_or_default()
1999                .iter()
2000                .map(|u| format!("{:?}", u.action()))
2001                .collect();
2002            actions.join(",")
2003        });
2004        let (ctx, client) = uf_context(&hooks, Vec::new());
2005        client_cell.set(Arc::clone(&client)).expect("set once");
2006        // The body starts before its START is written, so it returns once
2007        // the client has the START: the end hook then sees it.
2008        let body_client = Arc::clone(&client);
2009        let value = ctx
2010            .step(move |_| async move {
2011                for _ in 0..1000 {
2012                    if update_of(&body_client, "s", &OperationAction::Start).is_some() {
2013                        return Ok(1_u32);
2014                    }
2015                    tokio::task::yield_now().await;
2016                }
2017                Err("the START was never written".into())
2018            })
2019            .name("s")
2020            .await
2021            .unwrap();
2022        assert_eq!(value, 1);
2023        assert_eq!(hooks.find("s", "uf_end").probe, "Start");
2024        assert!(update_of(&client, "s", &OperationAction::Succeed).is_some());
2025    }
2026
2027    /// A failed body's wire is derived once, in the runner: the error's
2028    /// `Display` runs once, and the end hook's wire is the wire of
2029    /// `on_attempt_end`, where it fires, and of the FAIL or flat
2030    /// `BatchItem`.
2031    #[tokio::test]
2032    async fn failed_body_wire_derived_once() {
2033        use crate::plugin::recorder::Counted;
2034
2035        // A step that the retry strategy stops.
2036        let hooks = UfHooks::default();
2037        let (ctx, client) = uf_context(&hooks, Vec::new());
2038        let dc = Counted::new();
2039        let body_dc = dc.clone();
2040        let err = ctx
2041            .step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
2042            .name("step")
2043            .retry_strategy(|_err, _attempt| RetryDecision::Stop)
2044            .await
2045            .unwrap_err();
2046        drop(err);
2047        assert_eq!(dc.calls(), 1, "step");
2048        let end = hooks.find("step", "uf_end").wire.expect("an Err outcome");
2049        let attempt_end = hooks.find("step", "attempt_end").wire.expect("an error");
2050        assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
2051        let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
2052        assert_eq!(type_and_message(&end), update_error(&fail));
2053        assert_eq!(end.error_message(), Some("counted"));
2054
2055        // A child whose body returns `Err`.
2056        let hooks = UfHooks::default();
2057        let (ctx, client) = uf_context(&hooks, Vec::new());
2058        let dc = Counted::new();
2059        let body_dc = dc.clone();
2060        let err = ctx
2061            .run_in_child_context(
2062                move |_child| async move { Err::<u32, BoxError>(body_dc.boxed()) },
2063            )
2064            .name("child")
2065            .await
2066            .unwrap_err();
2067        drop(err);
2068        assert_eq!(dc.calls(), 1, "child");
2069        let end = hooks.find("child", "uf_end").wire.expect("an Err outcome");
2070        let fail = update_of(&client, "child", &OperationAction::Fail).expect("a FAIL");
2071        assert_eq!(type_and_message(&end), update_error(&fail));
2072
2073        // A `wait_for_callback` submitter that the retry strategy stops.
2074        // The probe reads the count when each hook runs.
2075        let dc = Counted::new();
2076        let probe_dc = dc.clone();
2077        let hooks = UfHooks::probing(move || probe_dc.calls().to_string());
2078        let (ctx, client) = uf_context(&hooks, Vec::new());
2079        let body_dc = dc.clone();
2080        let err = ctx
2081            .wait_for_callback::<String>(move |_step_ctx, _callback_id| async move {
2082                Err(body_dc.boxed())
2083            })
2084            .name("cb")
2085            .submitter_retry(|_err, _attempt| RetryDecision::Stop)
2086            .await
2087            .unwrap_err();
2088        drop(err);
2089        // The submitter's wire is derived once: one call by its end hooks.
2090        assert_eq!(hooks.find("cb-submitter", "uf_end").probe, "1");
2091        assert_eq!(hooks.find("cb-submitter", "end").probe, "1");
2092        // The body's own wire (rule 11's `wait_for_callback` body row,
2093        // `extract_wire_error`) takes the submitter step's recorded
2094        // message, so it does not format the error again.
2095        assert_eq!(dc.calls(), 1, "the submitter only");
2096        assert_eq!(hooks.find("cb", "uf_end").probe, "1");
2097        let outer = hooks.find("cb", "uf_end").wire.expect("an Err outcome");
2098        assert_eq!(outer.error_message(), Some("counted"));
2099        let end = hooks
2100            .find("cb-submitter", "uf_end")
2101            .wire
2102            .expect("an Err outcome");
2103        let attempt_end = hooks
2104            .find("cb-submitter", "attempt_end")
2105            .wire
2106            .expect("an error");
2107        assert_eq!(type_and_message(&end), type_and_message(&attempt_end));
2108        assert_eq!(end.error_message(), Some("counted"));
2109        // The submitter step's record carries no name.
2110        let fail = client
2111            .recorded_updates()
2112            .into_iter()
2113            .find(|u| u.r#type() == &OperationType::Step && u.action() == &OperationAction::Fail)
2114            .expect("the submitter's FAIL");
2115        assert_eq!(type_and_message(&end), update_error(&fail));
2116
2117        // A `map` item, in both modes.
2118        map_item_wire_derived_once().await;
2119    }
2120
2121    /// A step body that returns a recorded operation error records that
2122    /// error's type, message, data and stack unchanged.
2123    #[tokio::test]
2124    async fn failed_body_records_a_propagated_record_unchanged() {
2125        let hooks = UfHooks::default();
2126        let (ctx, client) = uf_context(&hooks, Vec::new());
2127        let err = ctx
2128            .step(|_| async {
2129                let wire = crate::error::WireError::new(Some("InnerType"), Some("inner"))
2130                    .with_error_data(Some("inner-data"))
2131                    .with_stack_trace(vec!["inner-frame".to_owned()]);
2132                Err::<u32, BoxError>(Box::new(crate::error::recorded_step_error(wire)))
2133            })
2134            .name("step")
2135            .retry_strategy(|_err, _attempt| RetryDecision::Stop)
2136            .await
2137            .unwrap_err();
2138        drop(err);
2139        let fail = update_of(&client, "step", &OperationAction::Fail).expect("a FAIL");
2140        let error = fail.error().expect("the FAIL carries an error object");
2141        assert_eq!(error.error_type(), Some("InnerType"));
2142        assert_eq!(error.error_message(), Some("inner"));
2143        assert_eq!(error.error_data(), Some("inner-data"));
2144        assert_eq!(error.stack_trace(), ["inner-frame"]);
2145    }
2146
2147    /// The `map` item case of [`failed_body_wire_derived_once`], in
2148    /// both modes.
2149    async fn map_item_wire_derived_once() {
2150        use crate::builders::map_parallel::CompletionConfig;
2151        use crate::map_parallel::NestingMode;
2152        use crate::plugin::recorder::Counted;
2153
2154        for mode in [NestingMode::Normal, NestingMode::Flat] {
2155            let hooks = UfHooks::default();
2156            let (ctx, client) = uf_context(&hooks, Vec::new());
2157            let dc = Counted::new();
2158            let body_dc = dc.clone();
2159            let batch = ctx
2160                .map(vec![1_u32], move |_child, _item, _idx| {
2161                    let dc = body_dc.clone();
2162                    async move { Err::<u32, BoxError>(dc.boxed()) }
2163                })
2164                .name("m")
2165                .nesting(mode)
2166                .completion(CompletionConfig::with_tolerated_failure_count(1))
2167                .await_batch()
2168                .await
2169                .unwrap();
2170            assert_eq!(dc.calls(), 1, "{mode:?}");
2171            let end = hooks
2172                .find("map-item-0", "uf_end")
2173                .wire
2174                .expect("an Err outcome");
2175            let item = batch.errors().next().expect("the item failed");
2176            assert_eq!(
2177                type_and_message(&end),
2178                (
2179                    item.error_type.map(str::to_owned),
2180                    Some(item.message.to_owned())
2181                ),
2182                "{mode:?}"
2183            );
2184            let fail = update_of(&client, "map-item-0", &OperationAction::Fail);
2185            match mode {
2186                NestingMode::Normal => {
2187                    assert_eq!(type_and_message(&end), update_error(&fail.expect("a FAIL")));
2188                }
2189                NestingMode::Flat => assert!(fail.is_none(), "a flat item writes no record"),
2190            }
2191        }
2192    }
2193
2194    /// Rule 8.2: the wire is derived under `ActiveBody::Outcome`, so an
2195    /// operation the error's `Display` claims fails the execution with a
2196    /// context usage error naming the step. That record precedes the
2197    /// step's `FAIL`, so the `FAIL` is refused and the step never resolves.
2198    #[tokio::test]
2199    async fn failing_step_display_calls_durable_op() {
2200        use crate::plugin::recorder::Counted;
2201        let hooks = UfHooks::default();
2202        let (ctx, client) = uf_context(&hooks, Vec::new());
2203        let claim_ctx = ctx.clone();
2204        let dc = Counted::calling(move || drop(claim_ctx.wait(Duration::from_secs(1))));
2205        let body_dc = dc.clone();
2206        let step = ctx
2207            .step(move |_| async move { Err::<u32, BoxError>(body_dc.boxed()) })
2208            .name("s")
2209            .retry_strategy(|_err, _attempt| RetryDecision::Stop);
2210        assert!(
2211            tokio::time::timeout(Duration::from_millis(50), step.into_future())
2212                .await
2213                .is_err(),
2214            "the step's FAIL is refused, so the step never resolves"
2215        );
2216        assert_eq!(dc.calls(), 1);
2217        assert!(
2218            update_of(&client, "s", &OperationAction::Fail).is_none(),
2219            "no terminal update is recorded"
2220        );
2221        assert!(update_of(&client, "s", &OperationAction::Succeed).is_none());
2222        let fatal = ctx.coordinator().fatal_error().expect("fatal recorded");
2223        assert!(
2224            fatal
2225                .error_message
2226                .contains("while the SDK records the outcome of operation \"1\""),
2227            "{}",
2228            fatal.error_message
2229        );
2230    }
2231
2232    /// A serdes that fails every serialization with a permanent error.
2233    struct FailingSerializer;
2234
2235    impl<T: Send + 'static> Serdes<T> for FailingSerializer {
2236        // reason: a test serdes that completes at once, in the async-fn form user code writes
2237        #[expect(clippy::unused_async_trait_impl)]
2238        async fn serialize(&self, _value: T, _context: SerdesContext) -> Result<String, BoxError> {
2239            Err("cannot encode".into())
2240        }
2241
2242        // reason: a test serdes that completes at once, in the async-fn form user code writes
2243        #[expect(clippy::unused_async_trait_impl)]
2244        async fn deserialize(&self, _wire: String, _context: SerdesContext) -> Result<T, BoxError> {
2245            Err("cannot decode".into())
2246        }
2247    }
2248
2249    /// Rule 11's step row: a body that succeeds ends `Ok`; the
2250    /// serialization failure after it records its own wire.
2251    #[tokio::test]
2252    async fn serialization_failure_keeps_its_own_wire() {
2253        let hooks = UfHooks::default();
2254        let (ctx, client) = uf_context(&hooks, Vec::new());
2255        let err = ctx
2256            .step(|_| async { Ok(1_u32) })
2257            .name("s")
2258            .serdes(FailingSerializer)
2259            .await
2260            .unwrap_err();
2261        drop(err);
2262        let end = hooks.find("s", "uf_end");
2263        assert_eq!(end.ok, Some(true));
2264        assert_eq!(end.wire, None);
2265        let fail = update_of(&client, "s", &OperationAction::Fail).expect("a FAIL");
2266        let (error_type, _) = update_error(&fail);
2267        assert_eq!(
2268            error_type.as_deref(),
2269            Some(crate::error::SERIALIZATION_FAILED_ERROR_TYPE)
2270        );
2271    }
2272
2273    /// Builds a step record for the first operation of a root context.
2274    fn step_record(
2275        status: CheckpointStatus,
2276        attempt: u32,
2277        error: Option<&crate::error::WireError>,
2278    ) -> (String, crate::engine::CheckpointRecord) {
2279        let wire_key = crate::engine::compute_wire_id_public("1");
2280        let record = crate::engine::CheckpointRecord {
2281            id: wire_key.clone(),
2282            status,
2283            result: None,
2284            error_type: error
2285                .and_then(crate::error::WireError::error_type)
2286                .map(str::to_owned),
2287            error_message: error
2288                .and_then(crate::error::WireError::error_message)
2289                .map(str::to_owned),
2290            error_data: error
2291                .and_then(crate::error::WireError::error_data)
2292                .map(str::to_owned),
2293            stack_trace: None,
2294            attempt,
2295            invoke_result: None,
2296            invoke_error_type: None,
2297            invoke_error_message: None,
2298            invoke_error_data: None,
2299            invoke_stack_trace: None,
2300            replay_children: false,
2301            callback_id: None,
2302            op_type: None,
2303            sub_type: None,
2304            due_at: None,
2305            op_name: None,
2306            parent_id: None,
2307            start_timestamp: None,
2308            end_timestamp: None,
2309        };
2310        (wire_key, record)
2311    }
2312
2313    /// Builds a root context over `records` that writes to a fresh
2314    /// in-memory client.
2315    fn context_over(
2316        records: Vec<(String, crate::engine::CheckpointRecord)>,
2317    ) -> (DurableContext, Arc<crate::client::InMemoryExecutionClient>) {
2318        let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
2319        let ctx = DurableContext::new_root_with_client(
2320            "arn:test".to_owned(),
2321            lambda_runtime::Context::default(),
2322            Arc::new(CheckpointLog::from_records(records)),
2323            Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
2324            "token0".to_owned(),
2325        );
2326        (ctx, client)
2327    }
2328
2329    /// The `FAIL` update a test context recorded, as a wire record.
2330    fn recorded_fail(client: &crate::client::InMemoryExecutionClient) -> crate::error::WireError {
2331        let updates = client.recorded_updates();
2332        let fail = updates
2333            .iter()
2334            .find(|u| matches!(u.action(), OperationAction::Fail))
2335            .expect("a FAIL update must be recorded");
2336        let error = fail
2337            .error()
2338            .expect("the FAIL update carries an error object");
2339        assert!(
2340            error.stack_trace().is_empty(),
2341            "the SDK captures no stack trace: {error:?}"
2342        );
2343        crate::error::WireError::new(error.error_type(), error.error_message())
2344            .with_error_data(error.error_data())
2345    }
2346
2347    /// A step with subtype `Payment` that fails attempt 1 with a 1 s retry
2348    /// and succeeds attempt 2.
2349    fn payment_step(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
2350        let op_id = ctx.mint_id();
2351        StepExecution {
2352            ctx,
2353            op_id,
2354            name: Some("pay".to_owned()),
2355            retry_strategy: Some(Box::new(|_err: &StepError, attempt: u32| {
2356                if attempt >= 2 {
2357                    RetryDecision::Stop
2358                } else {
2359                    RetryDecision::retry(Duration::from_secs(1))
2360                }
2361            })),
2362            serdes: crate::serdes::JsonSerdes,
2363            semantics: StepSemantics::default(),
2364            sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
2365            plugin_name: None,
2366            closure: |step_ctx: StepContext| async move {
2367                if step_ctx.attempt() == 1 {
2368                    Err::<i32, BoxError>("transient".into())
2369                } else {
2370                    Ok(7)
2371                }
2372            },
2373            _marker: std::marker::PhantomData,
2374        }
2375        .execute()
2376    }
2377
2378    /// A custom subtype is the `SubType` of every update the step writes:
2379    /// START and RETRY on attempt 1, START and SUCCEED on attempt 2. The
2380    /// in-memory client never moves the record to `READY`, so attempt 2
2381    /// runs in a second invocation over a `READY` record.
2382    #[tokio::test]
2383    async fn custom_sub_type_on_every_step_update() {
2384        // Attempt 1: the body fails, the strategy retries, and the step
2385        // suspends.
2386        let (ctx, first) = context_over(Vec::new());
2387        let signal = Arc::clone(ctx.coordinator());
2388        let outcome = crate::driver::test_support::outcome_of(
2389            ctx.clone(),
2390            Arc::clone(ctx.task_ownership()),
2391            signal,
2392            payment_step(ctx),
2393        )
2394        .await;
2395        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2396
2397        // Attempt 2 over the READY record: the body succeeds.
2398        let (id, mut record) = step_record(CheckpointStatus::Ready, 1, None);
2399        record.sub_type = Some("Payment".to_owned());
2400        let (ctx, second) = context_over(vec![(id, record)]);
2401        assert_eq!(payment_step(ctx).await.expect("attempt 2 succeeds"), 7);
2402
2403        let updates: Vec<OperationUpdate> = first
2404            .recorded_updates()
2405            .into_iter()
2406            .chain(second.recorded_updates())
2407            .collect();
2408        let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
2409        assert_eq!(
2410            actions,
2411            [
2412                OperationAction::Start,
2413                OperationAction::Retry,
2414                OperationAction::Start,
2415                OperationAction::Succeed
2416            ]
2417        );
2418        for update in &updates {
2419            assert_eq!(update.sub_type(), Some("Payment"), "{update:?}");
2420        }
2421    }
2422
2423    /// The SUCCEED write the service rejects carries the custom subtype,
2424    /// and no terminal FAIL follows it: the step's record stays at its
2425    /// accepted START.
2426    #[tokio::test]
2427    async fn custom_sub_type_on_rejected_write_and_no_terminal_fail() {
2428        use crate::client::TestResponse;
2429
2430        let (ctx, client) = context_over(Vec::new());
2431        client.enqueue_checkpoint_response(TestResponse::Success(vec![]));
2432        client.enqueue_checkpoint_response(TestResponse::NonRetryableError(
2433            "payload rejected".to_owned(),
2434        ));
2435        let op_id = ctx.mint_id();
2436        let exec = StepExecution {
2437            ctx: ctx.clone(),
2438            op_id,
2439            name: Some("pay".to_owned()),
2440            retry_strategy: None,
2441            serdes: crate::serdes::JsonSerdes,
2442            semantics: StepSemantics::default(),
2443            sub_type: OperationSubType::new("Payment").expect("a valid subtype"),
2444            plugin_name: None,
2445            closure: |_: StepContext| async { Ok::<i32, BoxError>(7) },
2446            _marker: std::marker::PhantomData,
2447        };
2448        let signal = Arc::clone(ctx.coordinator());
2449        let outcome = crate::driver::test_support::outcome_of(
2450            ctx.clone(),
2451            Arc::clone(ctx.task_ownership()),
2452            signal,
2453            exec.execute(),
2454        )
2455        .await;
2456        assert!(
2457            matches!(outcome, crate::driver::InvocationOutcome::Failed { .. }),
2458            "a rejected SUCCEED fails the execution: {outcome:?}"
2459        );
2460
2461        let updates = client.recorded_updates();
2462        let actions: Vec<OperationAction> = updates.iter().map(|u| u.action().clone()).collect();
2463        assert_eq!(actions, [OperationAction::Start, OperationAction::Succeed]);
2464        let rejected = updates.last().expect("the rejected SUCCEED is recorded");
2465        assert_eq!(rejected.sub_type(), Some("Payment"));
2466    }
2467
2468    /// A step body that fails with a typed error carrying error data.
2469    async fn typed_failing_body(_: StepContext) -> Result<i32, BoxError> {
2470        Err(Box::new(
2471            crate::error::TypedError::with_type("PaymentError", "declined")
2472                .with_error_data(r#"{"code":42}"#),
2473        ))
2474    }
2475
2476    #[tokio::test]
2477    async fn step_failure_records_typed_error_data() {
2478        // Live: the FAIL update carries the type and the data.
2479        let (ctx, client) = context_over(Vec::new());
2480        let op_id = ctx.mint_id();
2481        let exec = StepExecution {
2482            ctx,
2483            op_id,
2484            name: None,
2485            retry_strategy: Some(Box::new(|_err: &StepError, _attempt: u32| {
2486                RetryDecision::Stop
2487            })),
2488            serdes: crate::serdes::JsonSerdes,
2489            semantics: StepSemantics::default(),
2490            sub_type: OperationSubType::STEP,
2491            plugin_name: None,
2492            closure: typed_failing_body,
2493            _marker: std::marker::PhantomData,
2494        };
2495        let live = exec.execute().await.expect_err("the step must fail");
2496        let recorded = recorded_fail(&client);
2497        assert_eq!(recorded.error_type(), Some("PaymentError"));
2498        assert_eq!(recorded.error_message(), Some("declined"));
2499        assert_eq!(recorded.error_data(), Some(r#"{"code":42}"#));
2500        assert_eq!(
2501            live.wire().and_then(crate::error::WireError::error_data),
2502            Some(r#"{"code":42}"#)
2503        );
2504        assert_eq!(
2505            live.wire().and_then(crate::error::WireError::error_message),
2506            Some("declined")
2507        );
2508
2509        // Replay: the recorded data is reachable through `wire()`.
2510        let replayed_wire = crate::error::WireError::new(Some("PaymentError"), Some("declined"))
2511            .with_error_data(Some(r#"{"code":42}"#));
2512        let (ctx, _client) = context_over(vec![step_record(
2513            CheckpointStatus::Failed,
2514            0,
2515            Some(&replayed_wire),
2516        )]);
2517        let op_id = ctx.mint_id();
2518        let exec = StepExecution {
2519            ctx,
2520            op_id,
2521            name: None,
2522            retry_strategy: None,
2523            serdes: crate::serdes::JsonSerdes,
2524            semantics: StepSemantics::default(),
2525            sub_type: OperationSubType::STEP,
2526            plugin_name: None,
2527            closure: typed_failing_body,
2528            _marker: std::marker::PhantomData,
2529        };
2530        let err = exec
2531            .execute()
2532            .await
2533            .expect_err("the replayed step must fail");
2534        assert_eq!(
2535            err.wire().and_then(crate::error::WireError::error_data),
2536            Some(r#"{"code":42}"#)
2537        );
2538        assert_eq!(
2539            err.wire().and_then(crate::error::WireError::error_message),
2540            Some("declined")
2541        );
2542    }
2543
2544    #[tokio::test]
2545    async fn step_stop_returns_execution_failed_live_and_replay() {
2546        fn retry_once() -> RetryStrategy {
2547            Box::new(|_err: &StepError, attempt: u32| {
2548                if attempt >= 2 {
2549                    RetryDecision::Stop
2550                } else {
2551                    RetryDecision::retry(Duration::from_secs(1))
2552                }
2553            })
2554        }
2555        fn execution(ctx: DurableContext) -> impl Future<Output = Result<i32, OperationError>> {
2556            let op_id = ctx.mint_id();
2557            StepExecution {
2558                ctx,
2559                op_id,
2560                name: None,
2561                retry_strategy: Some(retry_once()),
2562                serdes: crate::serdes::JsonSerdes,
2563                semantics: StepSemantics::default(),
2564                sub_type: OperationSubType::STEP,
2565                plugin_name: None,
2566                closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
2567                _marker: std::marker::PhantomData,
2568            }
2569            .execute()
2570        }
2571        fn assert_execution_failed(err: &OperationError) {
2572            let OperationErrorKind::Step(step_err) = err.kind() else {
2573                unreachable!("a step builds a step error: {err:?}");
2574            };
2575            assert!(
2576                matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
2577                "unexpected kind: {:?}",
2578                step_err.kind()
2579            );
2580            let rendered = format!("{err:#}");
2581            assert!(rendered.ends_with("boom"), "got: {rendered}");
2582            assert!(!rendered.contains("retries exhausted"), "got: {rendered}");
2583        }
2584
2585        // Attempt 1: the strategy retries, so the step suspends.
2586        let (ctx, client) = context_over(Vec::new());
2587        let signal = Arc::clone(ctx.coordinator());
2588        let outcome = crate::driver::test_support::outcome_of(
2589            ctx.clone(),
2590            Arc::clone(ctx.task_ownership()),
2591            signal,
2592            execution(ctx),
2593        )
2594        .await;
2595        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2596        assert!(
2597            client
2598                .recorded_updates()
2599                .iter()
2600                .any(|u| matches!(u.action(), OperationAction::Retry)),
2601            "attempt 1 records a RETRY"
2602        );
2603
2604        // Attempt 2 (one completed retry): the strategy stops.
2605        let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Ready, 1, None)]);
2606        let live = execution(ctx).await.expect_err("the step must fail");
2607        assert_execution_failed(&live);
2608        assert_eq!(
2609            format!("{live:#}"),
2610            "operation error: step: execution failed: boom"
2611        );
2612
2613        // Replay of the recorded FAIL: the same kind and the same chain end.
2614        let recorded = recorded_fail(&client);
2615        let (ctx, _client) = context_over(vec![step_record(
2616            CheckpointStatus::Failed,
2617            1,
2618            Some(&recorded),
2619        )]);
2620        let replayed = execution(ctx)
2621            .await
2622            .expect_err("the replayed step must fail");
2623        assert_execution_failed(&replayed);
2624    }
2625
2626    /// Acceptance (issue #41): on the live path, an operation failure
2627    /// exposes the caller's concrete error type through a `source()`
2628    /// downcast: the escaping error is carried, not stringified.
2629    #[tokio::test]
2630    async fn step_live_failure_source_downcasts_to_concrete_user_type() {
2631        use crate::client::InMemoryExecutionClient;
2632
2633        #[derive(Debug)]
2634        struct PaymentDeclined {
2635            code: u16,
2636        }
2637        impl std::fmt::Display for PaymentDeclined {
2638            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2639                write!(f, "payment declined (code {})", self.code)
2640            }
2641        }
2642        impl std::error::Error for PaymentDeclined {}
2643
2644        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2645        let log = Arc::new(CheckpointLog::empty());
2646        let ctx = DurableContext::new_root_with_client(
2647            "arn:test".to_owned(),
2648            lambda_runtime::Context::default(),
2649            log,
2650            client,
2651            "token0".to_owned(),
2652        );
2653        let op_id = ctx.mint_id();
2654
2655        let no_retry: RetryStrategy =
2656            Box::new(|_err: &StepError, _attempt: u32| RetryDecision::Stop);
2657        let closure = |_: StepContext| async {
2658            Err::<i32, BoxError>(Box::new(PaymentDeclined { code: 402 }))
2659        };
2660
2661        let exec = StepExecution {
2662            ctx,
2663            op_id,
2664            name: None,
2665            retry_strategy: Some(no_retry),
2666            serdes: crate::serdes::JsonSerdes,
2667            semantics: StepSemantics::default(),
2668            sub_type: OperationSubType::STEP,
2669            plugin_name: None,
2670            closure,
2671            _marker: std::marker::PhantomData,
2672        };
2673
2674        let err = exec.execute().await.expect_err("step must fail");
2675
2676        // Walk source() to the caller's concrete error and downcast it.
2677        let mut source: Option<&(dyn std::error::Error + 'static)> =
2678            std::error::Error::source(&err);
2679        let mut found = None;
2680        while let Some(e) = source {
2681            if let Some(declined) = e.downcast_ref::<PaymentDeclined>() {
2682                found = Some(declined);
2683                break;
2684            }
2685            source = e.source();
2686        }
2687        let declined = found.expect("caller's concrete error type must be reachable via source()");
2688        assert_eq!(declined.code, 402);
2689
2690        // The wire failure record is reachable from the error too.
2691        assert_eq!(err.status(), Some("FAILED"));
2692        let wire = err.wire().expect("live failure carries its wire record");
2693        assert_eq!(wire.error_type(), Some("Error"));
2694        assert!(
2695            wire.error_message()
2696                .is_some_and(|m| m.contains("payment declined (code 402)")),
2697            "wire message flattens the chain: {wire:?}"
2698        );
2699        assert!(
2700            wire.stack_trace().is_empty(),
2701            "the SDK captures no stack trace for a live failure"
2702        );
2703    }
2704
2705    #[tokio::test]
2706    async fn step_retry_schedules_and_suspends() {
2707        use crate::client::InMemoryExecutionClient;
2708
2709        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2710        let log = Arc::new(CheckpointLog::empty());
2711        let ctx = DurableContext::new_root_with_client(
2712            "arn:test".to_owned(),
2713            lambda_runtime::Context::default(),
2714            log,
2715            client,
2716            "token0".to_owned(),
2717        );
2718        let op_id = ctx.mint_id();
2719
2720        // Strategy that always retries with 1s delay.
2721        let always_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
2722            RetryDecision::retry(Duration::from_secs(1))
2723        });
2724
2725        let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
2726
2727        let exec = StepExecution {
2728            ctx: ctx.clone(),
2729            op_id,
2730            name: None,
2731            retry_strategy: Some(always_retry),
2732            serdes: crate::serdes::JsonSerdes,
2733            semantics: StepSemantics::default(),
2734            sub_type: OperationSubType::STEP,
2735            plugin_name: None,
2736            closure,
2737            _marker: std::marker::PhantomData,
2738        };
2739
2740        let signal = Arc::clone(ctx.coordinator());
2741        // Retry timer is backend-owned: the operation suspends (parks) rather
2742        // than surfacing a fabricated error to the caller.
2743        let outcome = crate::driver::test_support::outcome_of(
2744            ctx.clone(),
2745            Arc::clone(ctx.task_ownership()),
2746            signal,
2747            exec.execute(),
2748        )
2749        .await;
2750        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2751    }
2752
2753    #[tokio::test]
2754    async fn step_retry_fractional_delay_rounds_up() {
2755        use crate::client::InMemoryExecutionClient;
2756
2757        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2758        let log = Arc::new(CheckpointLog::empty());
2759        let ctx = DurableContext::new_root_with_client(
2760            "arn:test".to_owned(),
2761            lambda_runtime::Context::default(),
2762            log,
2763            client.clone(),
2764            "token0".to_owned(),
2765        );
2766        let op_id = ctx.mint_id();
2767
2768        // Strategy that retries with a fractional 1.9s delay. Truncation
2769        // would checkpoint 1s and retry EARLIER than requested; ceiling
2770        // semantics must checkpoint 2s.
2771        let fractional_retry: RetryStrategy = Box::new(|_err: &StepError, _attempt: u32| {
2772            RetryDecision::retry(Duration::from_millis(1900))
2773        });
2774
2775        let closure = |_: StepContext| async { Err::<i32, BoxError>("transient".into()) };
2776
2777        let exec = StepExecution {
2778            ctx: ctx.clone(),
2779            op_id,
2780            name: None,
2781            retry_strategy: Some(fractional_retry),
2782            serdes: crate::serdes::JsonSerdes,
2783            semantics: StepSemantics::default(),
2784            sub_type: OperationSubType::STEP,
2785            plugin_name: None,
2786            closure,
2787            _marker: std::marker::PhantomData,
2788        };
2789
2790        let signal = Arc::clone(ctx.coordinator());
2791        let outcome = crate::driver::test_support::outcome_of(
2792            ctx.clone(),
2793            Arc::clone(ctx.task_ownership()),
2794            signal,
2795            exec.execute(),
2796        )
2797        .await;
2798        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
2799
2800        // The RETRY update must carry the rounded-up delay.
2801        let updates = client.recorded_updates();
2802        let retry = updates
2803            .iter()
2804            .find(|u| matches!(u.action(), OperationAction::Retry))
2805            .expect("a Retry update must be recorded");
2806        let delay = retry
2807            .step_options()
2808            .and_then(aws_sdk_lambda::types::StepOptions::next_attempt_delay_seconds)
2809            .expect("Retry update must carry NextAttemptDelaySeconds");
2810        assert_eq!(delay, 2, "1.9s must round UP to 2s, not truncate to 1s");
2811    }
2812
2813    #[tokio::test]
2814    async fn step_spawn_executes_on_blessed_task() {
2815        use crate::client::InMemoryExecutionClient;
2816
2817        let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2818        let log = Arc::new(CheckpointLog::empty());
2819        let ctx = DurableContext::new_root_with_client(
2820            "arn:test".to_owned(),
2821            lambda_runtime::Context::default(),
2822            log,
2823            client,
2824            "token0".to_owned(),
2825        );
2826
2827        // Use ctx.step() which wires up the closure properly.
2828        let handle = ctx.step(|_| async { Ok(7i32) }).name("spawned").spawn();
2829        let result = handle.await;
2830        assert_eq!(result.unwrap(), 7);
2831    }
2832
2833    #[tokio::test]
2834    async fn step_ownership_rejection_from_foreign_task() {
2835        use crate::client::InMemoryExecutionClient;
2836
2837        // Must create the context inside a spawned task where try_id()
2838        // returns Some: #[tokio::test] root runs in block_on with no task ID.
2839        let result = tokio::spawn(async {
2840            let client = Arc::new(InMemoryExecutionClient::new(Vec::new()));
2841            let log = Arc::new(CheckpointLog::empty());
2842            let ctx = DurableContext::new_root_with_client(
2843                "arn:test".to_owned(),
2844                lambda_runtime::Context::default(),
2845                log,
2846                client,
2847                "token0".to_owned(),
2848            );
2849
2850            // Spawn a DIFFERENT (non-blessed) task and try executing a step.
2851            let ctx_clone = ctx.clone();
2852            let handle = tokio::spawn(async move {
2853                let op_id = ctx_clone.mint_id();
2854                let closure = |_: StepContext| async { Ok::<i32, BoxError>(1) };
2855
2856                let exec = StepExecution {
2857                    ctx: ctx_clone,
2858                    op_id,
2859                    name: None,
2860                    retry_strategy: None,
2861                    serdes: crate::serdes::JsonSerdes,
2862                    semantics: StepSemantics::default(),
2863                    sub_type: OperationSubType::STEP,
2864                    plugin_name: None,
2865                    closure,
2866                    _marker: std::marker::PhantomData,
2867                };
2868                exec.execute().await
2869            });
2870
2871            handle.await.unwrap()
2872        })
2873        .await;
2874
2875        let inner_result = result.unwrap();
2876        assert!(inner_result.is_err());
2877        let err_msg = format!("{:#}", inner_result.unwrap_err());
2878        assert!(
2879            err_msg.contains("task") || err_msg.contains("ownership"),
2880            "expected ownership error, got: {err_msg}"
2881        );
2882    }
2883
2884    // AtMostOncePerRetry tests
2885
2886    /// A strategy that records the display of each kind it sees and
2887    /// returns `decision`.
2888    fn kind_recording_strategy(
2889        decision: RetryDecision,
2890    ) -> (RetryStrategy, Arc<std::sync::Mutex<Vec<String>>>) {
2891        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2892        let seen_clone = Arc::clone(&seen);
2893        let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
2894            seen_clone
2895                .lock()
2896                .expect("kind log lock")
2897                .push(err.kind().to_string());
2898            decision.clone()
2899        });
2900        (strategy, seen)
2901    }
2902
2903    /// The error object of the first update a test context recorded with
2904    /// `action`, as a wire record.
2905    fn recorded_error_for(
2906        client: &crate::client::InMemoryExecutionClient,
2907        action: &OperationAction,
2908    ) -> crate::error::WireError {
2909        let updates = client.recorded_updates();
2910        let update = updates
2911            .iter()
2912            .find(|u| u.action() == action)
2913            .unwrap_or_else(|| panic!("a {action:?} update must be recorded: {updates:?}"));
2914        let error = update.error().expect("the update carries an error object");
2915        crate::error::WireError::new(error.error_type(), error.error_message())
2916    }
2917
2918    /// An `AtMostOncePerRetry` step over a STARTED record, whose body
2919    /// sets `executed` if it runs.
2920    fn at_most_once_over_started(
2921        ctx: DurableContext,
2922        retry_strategy: Option<RetryStrategy>,
2923        executed: Arc<std::sync::atomic::AtomicBool>,
2924    ) -> StepExecution<
2925        i32,
2926        impl FnOnce(StepContext) -> std::future::Ready<Result<i32, BoxError>> + Clone,
2927        crate::serdes::JsonSerdes,
2928    > {
2929        let op_id = ctx.mint_id();
2930        StepExecution {
2931            ctx,
2932            op_id,
2933            name: Some("at-most-once-step".to_owned()),
2934            retry_strategy,
2935            serdes: crate::serdes::JsonSerdes,
2936            semantics: StepSemantics::AtMostOncePerRetry,
2937            sub_type: OperationSubType::STEP,
2938            plugin_name: None,
2939            closure: move |_: StepContext| {
2940                executed.store(true, std::sync::atomic::Ordering::SeqCst);
2941                std::future::ready(Ok::<i32, BoxError>(42))
2942            },
2943            _marker: std::marker::PhantomData,
2944        }
2945    }
2946
2947    #[tokio::test]
2948    async fn step_at_most_once_replay_started_no_retry_fails_permanently() {
2949        use std::sync::atomic::{AtomicBool, Ordering};
2950
2951        let executed = Arc::new(AtomicBool::new(false));
2952        let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
2953        let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
2954        let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
2955
2956        let err = exec
2957            .execute()
2958            .await
2959            .expect_err("expected failure for interrupted + no retry");
2960
2961        assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
2962        let fail = recorded_error_for(&client, &OperationAction::Fail);
2963        assert_eq!(fail.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
2964        assert_eq!(fail.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
2965        let OperationErrorKind::Step(step_err) = err.kind() else {
2966            panic!("expected a step error, got {err:?}");
2967        };
2968        assert!(
2969            matches!(step_err.kind(), StepErrorKind::Interrupted),
2970            "got {step_err:?}"
2971        );
2972        assert_eq!(
2973            err.wire().and_then(crate::error::WireError::error_type),
2974            Some(STEP_INTERRUPTED_ERROR_TYPE)
2975        );
2976        assert!(
2977            !executed.load(Ordering::SeqCst),
2978            "closure should NOT execute when AtMostOncePerRetry and replay=Started"
2979        );
2980    }
2981
2982    #[tokio::test]
2983    async fn interrupted_attempt_gives_strategy_step_error_interrupted() {
2984        let executed = Arc::new(std::sync::atomic::AtomicBool::new(false));
2985        let (ctx, _client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
2986        let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
2987        let seen_clone = Arc::clone(&seen);
2988        let strategy: RetryStrategy = Box::new(move |err: &StepError, _attempt: u32| {
2989            let source = std::error::Error::source(err).map(ToString::to_string);
2990            seen_clone
2991                .lock()
2992                .expect("strategy log lock")
2993                .push((err.kind().to_string(), source));
2994            RetryDecision::Stop
2995        });
2996        let exec = at_most_once_over_started(ctx, Some(strategy), executed);
2997
2998        exec.execute()
2999            .await
3000            .expect_err("an interrupted attempt with no retry fails");
3001
3002        let expected = StepError::interrupted();
3003        assert_eq!(
3004            *seen.lock().expect("strategy log lock"),
3005            [(
3006                expected.kind().to_string(),
3007                std::error::Error::source(&expected).map(ToString::to_string),
3008            )]
3009        );
3010    }
3011
3012    #[tokio::test]
3013    async fn step_at_most_once_replay_started_with_retry_suspends() {
3014        use std::sync::atomic::{AtomicBool, Ordering};
3015
3016        let executed = Arc::new(AtomicBool::new(false));
3017        let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
3018        let (strategy, seen) =
3019            kind_recording_strategy(RetryDecision::retry(Duration::from_secs(5)));
3020        let signal = Arc::clone(ctx.coordinator());
3021        let ownership = Arc::clone(ctx.task_ownership());
3022        let root = ctx.clone();
3023        let exec = at_most_once_over_started(ctx, Some(strategy), Arc::clone(&executed));
3024
3025        // Retry timer is backend-owned: the operation suspends (parks) rather
3026        // than surfacing a fabricated retry-schedule error to the caller.
3027        let outcome =
3028            crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
3029        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
3030        assert_eq!(*seen.lock().expect("kind log lock"), ["step interrupted"]);
3031        let retry = recorded_error_for(&client, &OperationAction::Retry);
3032        assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3033        assert_eq!(retry.error_message(), Some(STEP_INTERRUPTED_MESSAGE));
3034        assert!(
3035            !executed.load(Ordering::SeqCst),
3036            "closure should NOT execute when AtMostOncePerRetry and replay=Started"
3037        );
3038    }
3039
3040    #[tokio::test]
3041    async fn step_at_most_once_interrupted_default_strategy_retries() {
3042        use std::sync::atomic::AtomicBool;
3043
3044        let executed = Arc::new(AtomicBool::new(false));
3045        let (ctx, client) = context_over(vec![step_record(CheckpointStatus::Started, 0, None)]);
3046        let signal = Arc::clone(ctx.coordinator());
3047        let ownership = Arc::clone(ctx.task_ownership());
3048        let root = ctx.clone();
3049        let exec = at_most_once_over_started(ctx, None, executed);
3050
3051        let outcome =
3052            crate::driver::test_support::outcome_of(root, ownership, signal, exec.execute()).await;
3053        assert_eq!(outcome, crate::driver::InvocationOutcome::Pending);
3054        let retry = recorded_error_for(&client, &OperationAction::Retry);
3055        assert_eq!(retry.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3056    }
3057
3058    #[tokio::test]
3059    async fn step_body_error_strategy_sees_execution_failed() {
3060        let (ctx, client) = context_over(Vec::new());
3061        let op_id = ctx.mint_id();
3062        let (strategy, seen) = kind_recording_strategy(RetryDecision::Stop);
3063        let exec = StepExecution {
3064            ctx,
3065            op_id,
3066            name: None,
3067            retry_strategy: Some(strategy),
3068            serdes: crate::serdes::JsonSerdes,
3069            semantics: StepSemantics::AtMostOncePerRetry,
3070            sub_type: OperationSubType::STEP,
3071            plugin_name: None,
3072            closure: |_: StepContext| async { Err::<i32, BoxError>("boom".into()) },
3073            _marker: std::marker::PhantomData,
3074        };
3075
3076        let err = exec.execute().await.expect_err("the step must fail");
3077
3078        assert_eq!(*seen.lock().expect("kind log lock"), ["execution failed"]);
3079        let fail = recorded_error_for(&client, &OperationAction::Fail);
3080        assert_eq!(fail.error_type(), Some(STEP_FALLBACK_ERROR_TYPE));
3081        let OperationErrorKind::Step(step_err) = err.kind() else {
3082            panic!("expected a step error, got {err:?}");
3083        };
3084        assert!(
3085            matches!(step_err.kind(), StepErrorKind::ExecutionFailed),
3086            "got {step_err:?}"
3087        );
3088    }
3089
3090    #[tokio::test]
3091    async fn step_interrupted_fail_record_replays_as_interrupted() {
3092        let wire = crate::error::WireError::new(
3093            Some(STEP_INTERRUPTED_ERROR_TYPE),
3094            Some(STEP_INTERRUPTED_MESSAGE),
3095        );
3096        let (ctx, _client) =
3097            context_over(vec![step_record(CheckpointStatus::Failed, 1, Some(&wire))]);
3098        let op_id = ctx.mint_id();
3099        let exec = StepExecution {
3100            ctx,
3101            op_id,
3102            name: None,
3103            retry_strategy: None,
3104            serdes: crate::serdes::JsonSerdes,
3105            semantics: StepSemantics::AtMostOncePerRetry,
3106            sub_type: OperationSubType::STEP,
3107            plugin_name: None,
3108            closure: |_: StepContext| async { Ok::<i32, BoxError>(1) },
3109            _marker: std::marker::PhantomData,
3110        };
3111
3112        let err = exec
3113            .execute()
3114            .await
3115            .expect_err("the replayed step must fail");
3116
3117        let OperationErrorKind::Step(step_err) = err.kind() else {
3118            panic!("expected a step error, got {err:?}");
3119        };
3120        assert!(
3121            matches!(step_err.kind(), StepErrorKind::Interrupted),
3122            "got {step_err:?}"
3123        );
3124        assert_eq!(
3125            err.wire().and_then(crate::error::WireError::error_type),
3126            Some(STEP_INTERRUPTED_ERROR_TYPE)
3127        );
3128        let replayed = std::error::Error::source(step_err)
3129            .and_then(|s| s.downcast_ref::<crate::error::ReplayedFailure>())
3130            .expect("replay source is a ReplayedFailure");
3131        assert_eq!(replayed.error_type(), Some(STEP_INTERRUPTED_ERROR_TYPE));
3132    }
3133
3134    #[tokio::test]
3135    async fn step_at_least_once_replay_started_re_executes() {
3136        // Verify that the DEFAULT semantics still re-execute on Started replay.
3137        use crate::engine::{CheckpointLog, CheckpointRecord};
3138        use std::sync::atomic::{AtomicBool, Ordering};
3139
3140        let executed = Arc::new(AtomicBool::new(false));
3141        let executed_clone = Arc::clone(&executed);
3142
3143        let wire_key = crate::engine::compute_wire_id_public("1");
3144        let record = CheckpointRecord {
3145            id: wire_key.clone(),
3146            status: CheckpointStatus::Started,
3147            result: None,
3148            error_type: None,
3149            error_message: None,
3150            error_data: None,
3151            stack_trace: None,
3152            attempt: 0,
3153            invoke_result: None,
3154            invoke_error_type: None,
3155            invoke_error_message: None,
3156            invoke_error_data: None,
3157            invoke_stack_trace: None,
3158            replay_children: false,
3159            callback_id: None,
3160            op_type: None,
3161            sub_type: None,
3162            due_at: None,
3163            op_name: None,
3164            parent_id: None,
3165            start_timestamp: None,
3166            end_timestamp: None,
3167        };
3168        let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
3169
3170        let client = Arc::new(crate::client::InMemoryExecutionClient::new(Vec::new()));
3171        let ctx = DurableContext::new_root_with_client(
3172            "arn:test".to_owned(),
3173            lambda_runtime::Context::default(),
3174            log,
3175            client,
3176            "token0".to_owned(),
3177        );
3178        let op_id = ctx.mint_id();
3179
3180        let closure = move |_: StepContext| {
3181            executed_clone.store(true, Ordering::SeqCst);
3182            async { Ok::<i32, BoxError>(77) }
3183        };
3184
3185        let exec = StepExecution {
3186            ctx,
3187            op_id,
3188            name: None,
3189            retry_strategy: None,
3190            serdes: crate::serdes::JsonSerdes,
3191            semantics: StepSemantics::AtLeastOncePerRetry,
3192            sub_type: OperationSubType::STEP,
3193            plugin_name: None,
3194            closure,
3195            _marker: std::marker::PhantomData,
3196        };
3197
3198        let result = exec.execute().await;
3199        assert_eq!(result.unwrap(), 77);
3200        assert!(
3201            executed.load(Ordering::SeqCst),
3202            "closure SHOULD execute under AtLeastOncePerRetry with Started replay"
3203        );
3204    }
3205
3206    // RetryStrategyConfig delay shaping
3207
3208    use crate::builders::{Backoff, JitterStrategy, RetryStrategyConfig};
3209
3210    fn sample_error() -> StepError {
3211        StepError::new(StepErrorKind::ExecutionFailed, Some("boom".into()))
3212    }
3213
3214    /// Extracts the retry delay in whole seconds, panicking on `Stop`.
3215    fn retry_secs(decision: &RetryDecision) -> u64 {
3216        match decision {
3217            RetryDecision::Retry { delay } => delay.as_secs(),
3218            RetryDecision::Stop => panic!("expected Retry, got Stop"),
3219        }
3220    }
3221
3222    /// The default config with jitter disabled reproduces the documented
3223    /// constants attempt by attempt: 5s initial delay doubling to a 60s
3224    /// cap, stopping at the 6th attempt. This pins the deterministic
3225    /// backbone that `default_retry_strategy` jitters over.
3226    #[test]
3227    fn default_config_without_jitter_matches_documented_constants() {
3228        let config = RetryStrategyConfig::builder()
3229            .max_attempts(6)
3230            .max_delay(Duration::from_mins(1))
3231            .jitter(JitterStrategy::None)
3232            .build();
3233
3234        // initial 5s, rate 2.0: 5, 10, 20, 40, then capped at 60.
3235        let expected = [5u64, 10, 20, 40, 60];
3236        for (attempt, expected_secs) in (1u32..).zip(expected) {
3237            assert_eq!(
3238                retry_secs(&config.decide(&sample_error(), attempt)),
3239                expected_secs,
3240                "attempt {attempt}"
3241            );
3242        }
3243        assert_eq!(config.decide(&sample_error(), 6), RetryDecision::Stop);
3244        assert_eq!(config.decide(&sample_error(), 7), RetryDecision::Stop);
3245    }
3246
3247    /// `default_retry_strategy` agrees with `RetryStrategyConfig::default_preset`
3248    /// attempt by attempt, modulo jitter: it stops at exactly the same
3249    /// attempts, and each retry delay falls within the full-jitter envelope
3250    /// `[1, base]` where `base` is the no-jitter delay for that attempt.
3251    #[test]
3252    fn default_retry_strategy_matches_default_config_modulo_jitter() {
3253        let strategy = default_retry_strategy();
3254        let no_jitter = RetryStrategyConfig::builder()
3255            .max_attempts(6)
3256            .max_delay(Duration::from_mins(1))
3257            .jitter(JitterStrategy::None)
3258            .build();
3259        let err = sample_error();
3260
3261        for attempt in 1u32..=5 {
3262            // Sample repeatedly: full jitter randomizes within [1, base].
3263            let base = retry_secs(&no_jitter.decide(&sample_error(), attempt));
3264            for _ in 0..50 {
3265                let secs = retry_secs(&strategy(&err, attempt));
3266                assert!(
3267                    (1..=base).contains(&secs),
3268                    "attempt {attempt}: delay {secs}s outside full-jitter envelope [1, {base}]"
3269                );
3270            }
3271        }
3272        // Both stop at the same attempt boundary (6 total attempts).
3273        assert_eq!(strategy(&err, 6), RetryDecision::Stop);
3274        assert_eq!(no_jitter.decide(&sample_error(), 6), RetryDecision::Stop);
3275        assert_eq!(strategy(&err, 7), RetryDecision::Stop);
3276    }
3277
3278    /// Half jitter keeps every sampled delay within `[base / 2, base]`.
3279    #[test]
3280    fn half_jitter_delays_stay_within_bounds() {
3281        let config = RetryStrategyConfig::builder()
3282            .max_attempts(6)
3283            .initial_delay(Duration::from_secs(8))
3284            .max_delay(Duration::from_secs(64))
3285            .backoff_rate(2.0)
3286            .jitter(JitterStrategy::Half)
3287            .build();
3288
3289        for attempt in 1u32..=3 {
3290            // base: 8, 16, 32: half-jitter bounds [4, 8], [8, 16], [16, 32].
3291            let base = 8u64 << (attempt - 1);
3292            for _ in 0..100 {
3293                let secs = retry_secs(&config.decide(&sample_error(), attempt));
3294                assert!(
3295                    (base / 2..=base).contains(&secs),
3296                    "attempt {attempt}: delay {secs}s outside half-jitter bounds \
3297                     [{}, {base}]",
3298                    base / 2
3299                );
3300            }
3301        }
3302    }
3303
3304    /// The computed delay is capped at `max_delay` and never drops below
3305    /// one second, whatever the configuration.
3306    #[test]
3307    fn delays_are_capped_and_have_one_second_floor() {
3308        // A sub-second initial delay rises to the 1s floor.
3309        let tiny = RetryStrategyConfig::builder()
3310            .initial_delay(Duration::from_millis(1))
3311            .jitter(JitterStrategy::None)
3312            .build();
3313        assert_eq!(retry_secs(&tiny.decide(&sample_error(), 1)), 1);
3314
3315        // A huge backoff rate hits the cap immediately.
3316        let capped = RetryStrategyConfig::builder()
3317            .initial_delay(Duration::from_secs(30))
3318            .max_delay(Duration::from_secs(45))
3319            .backoff_rate(100.0)
3320            .jitter(JitterStrategy::None)
3321            .build();
3322        assert_eq!(retry_secs(&capped.decide(&sample_error(), 1)), 30);
3323        assert_eq!(retry_secs(&capped.decide(&sample_error(), 2)), 45);
3324    }
3325
3326    /// `max_attempts(1)` never retries: the first failure already reaches
3327    /// the attempt budget.
3328    #[test]
3329    fn max_attempts_one_never_retries() {
3330        let config = RetryStrategyConfig::builder().max_attempts(1).build();
3331        assert_eq!(config.decide(&sample_error(), 1), RetryDecision::Stop);
3332    }
3333
3334    /// Every row of the `quantize_delay` case table: round to nearest,
3335    /// one-second floor, and 1 second for a non-finite input. The rule is
3336    /// the JS rule `Math.max(1, Math.round(x))`.
3337    #[test]
3338    fn quantize_delay_rounds_to_nearest_with_one_second_floor() {
3339        let cases = [
3340            (0.0, 1u64),
3341            (0.4, 1),
3342            (1.1, 1),
3343            (2.25, 2),
3344            (2.5, 3),
3345            (4.4, 4),
3346            (5.0, 5),
3347            (f64::NAN, 1),
3348            (f64::INFINITY, 1),
3349        ];
3350        for (input, expected) in cases {
3351            assert_eq!(
3352                quantize_delay(input),
3353                Duration::from_secs(expected),
3354                "input {input}"
3355            );
3356        }
3357    }
3358
3359    /// Non-jittered fractional delays round to the nearest whole second,
3360    /// as in JS and Go: 1.1 s schedules 1 s, 2.2 s schedules 2 s, 2.25 s
3361    /// schedules 2 s, and 2.5 s schedules 3 s.
3362    #[test]
3363    fn fractional_delays_round_to_nearest() {
3364        let config = RetryStrategyConfig::builder()
3365            .initial_delay(Duration::from_millis(1100))
3366            .jitter(JitterStrategy::None)
3367            .build();
3368        assert_eq!(retry_secs(&config.decide(&sample_error(), 1)), 1);
3369        // 1.1 * 2 = 2.2 s.
3370        assert_eq!(retry_secs(&config.decide(&sample_error(), 2)), 2);
3371
3372        // 1 * 1.5^2 = 2.25 s.
3373        let slow = RetryStrategyConfig::builder()
3374            .max_attempts(6)
3375            .initial_delay(Duration::from_secs(1))
3376            .backoff_rate(1.5)
3377            .jitter(JitterStrategy::None)
3378            .build();
3379        assert_eq!(retry_secs(&slow.decide(&sample_error(), 3)), 2);
3380
3381        let half = RetryStrategyConfig::builder()
3382            .initial_delay(Duration::from_millis(2500))
3383            .jitter(JitterStrategy::None)
3384            .build();
3385        assert_eq!(retry_secs(&half.decide(&sample_error(), 1)), 3);
3386    }
3387
3388    /// Every row of the linear backoff case table, with no jitter. The
3389    /// JS formula is `min(initialDelay + increment * (attempt - 1),
3390    /// maxDelay)`, rounded to the nearest second with a 1 s floor. Row G
3391    /// pins the exponential default schedule, unchanged.
3392    #[test]
3393    fn linear_backoff_cases() {
3394        let linear = |initial: Duration, increment: Duration| {
3395            RetryStrategyConfig::builder()
3396                .initial_delay(initial)
3397                .backoff(Backoff::linear(increment))
3398                .jitter(JitterStrategy::None)
3399        };
3400        let delays = |config: &RetryStrategyConfig, attempts: std::ops::RangeInclusive<u32>| {
3401            attempts
3402                .map(|attempt| retry_secs(&config.decide(&sample_error(), attempt)))
3403                .collect::<Vec<_>>()
3404        };
3405
3406        // A: the preset's 1, 2, 3, 4, 5 s.
3407        let preset = RetryStrategyConfig::linear();
3408        assert_eq!(delays(&preset, 1..=5), [1, 2, 3, 4, 5], "case A");
3409        // B: the sixth failure is final.
3410        assert_eq!(
3411            preset.decide(&sample_error(), 6),
3412            RetryDecision::Stop,
3413            "case B"
3414        );
3415
3416        // C: 1, 11, then 21 capped at 15.
3417        let capped = linear(Duration::from_secs(1), Duration::from_secs(10))
3418            .max_delay(Duration::from_secs(15))
3419            .max_attempts(6)
3420            .build();
3421        assert_eq!(delays(&capped, 1..=3), [1, 11, 15], "case C");
3422
3423        // D: 1, 1.25, 1.5, 1.75 round to nearest: 1, 1, 2, 2.
3424        let fractional = linear(Duration::from_secs(1), Duration::from_millis(250))
3425            .max_attempts(6)
3426            .build();
3427        assert_eq!(delays(&fractional, 1..=4), [1, 1, 2, 2], "case D");
3428
3429        // E: a zero delay rises to the 1 s floor.
3430        let zero = linear(Duration::ZERO, Duration::ZERO)
3431            .max_attempts(3)
3432            .build();
3433        assert_eq!(delays(&zero, 1..=1), [1], "case E");
3434
3435        // F: no_retry stops on the first failure.
3436        assert_eq!(
3437            RetryStrategyConfig::no_retry().decide(&sample_error(), 1),
3438            RetryDecision::Stop,
3439            "case F"
3440        );
3441
3442        // G: the default exponential schedule without jitter, unchanged.
3443        let exponential = RetryStrategyConfig::builder()
3444            .max_attempts(6)
3445            .max_delay(Duration::from_mins(1))
3446            .jitter(JitterStrategy::None)
3447            .build();
3448        assert_eq!(delays(&exponential, 1..=5), [5, 10, 20, 40, 60], "case G");
3449    }
3450
3451    /// Half jitter on a linear backoff keeps every sampled delay within
3452    /// `[base / 2, base]`, with base 8, 16 and 24 s.
3453    #[test]
3454    fn linear_half_jitter_stays_within_bounds() {
3455        let config = RetryStrategyConfig::builder()
3456            .initial_delay(Duration::from_secs(8))
3457            .backoff(Backoff::linear(Duration::from_secs(8)))
3458            .jitter(JitterStrategy::Half)
3459            .build();
3460
3461        for (attempt, base) in [(1u32, 8u64), (2, 16), (3, 24)] {
3462            for _ in 0..100 {
3463                let secs = retry_secs(&config.decide(&sample_error(), attempt));
3464                assert!(
3465                    (base / 2..=base).contains(&secs),
3466                    "attempt {attempt}: delay {secs}s outside half-jitter bounds \
3467                     [{}, {base}]",
3468                    base / 2
3469                );
3470            }
3471        }
3472    }
3473
3474    /// A test error type for the `retry_if` case table.
3475    #[derive(Debug)]
3476    struct Transient;
3477
3478    impl std::fmt::Display for Transient {
3479        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3480            f.write_str("transient")
3481        }
3482    }
3483
3484    impl std::error::Error for Transient {}
3485
3486    fn is_transient(err: &StepError) -> bool {
3487        std::error::Error::source(err)
3488            .and_then(|e| e.downcast_ref::<Transient>())
3489            .is_some()
3490    }
3491
3492    fn mentions_timeout(err: &StepError) -> bool {
3493        std::error::Error::source(err)
3494            .map(ToString::to_string)
3495            .is_some_and(|message| message.contains("timeout"))
3496    }
3497
3498    fn body_error(source: crate::error::Source) -> StepError {
3499        StepError::new(StepErrorKind::ExecutionFailed, Some(source))
3500    }
3501
3502    /// Every row of the `retry_if` case table: the cap is checked first
3503    /// (the predicate is not called at the cap), then the predicate, then
3504    /// the unchanged delay schedule. Matching on the source's type
3505    /// (`instanceof` in JS), on its message (`includes` in JS), and on
3506    /// both with `||` (the OR of the two JS lists).
3507    #[test]
3508    fn retry_if_filter_cases() {
3509        use std::sync::Arc;
3510        use std::sync::atomic::{AtomicU32, Ordering};
3511
3512        type Predicate = fn(&StepError) -> bool;
3513        /// (case, predicate, error, attempt, expected delay secs, predicate called)
3514        type Case = (
3515            &'static str,
3516            Option<Predicate>,
3517            StepError,
3518            u32,
3519            Option<u64>,
3520            bool,
3521        );
3522        let either: Predicate = |err| is_transient(err) || mentions_timeout(err);
3523        let boom = || body_error("boom".into());
3524        let transient = || body_error(Box::new(Transient));
3525        let timeout = || body_error("read timeout".into());
3526
3527        let cases: Vec<Case> = vec![
3528            ("A", None, boom(), 1, Some(1), false),
3529            ("B", Some(|_| false), boom(), 1, None, true),
3530            ("C", Some(|_| true), boom(), 2, Some(2), true),
3531            ("D", Some(|_| true), boom(), 3, None, false),
3532            ("E", Some(is_transient), transient(), 1, Some(1), true),
3533            ("F", Some(is_transient), boom(), 1, None, true),
3534            ("G", Some(mentions_timeout), timeout(), 1, Some(1), true),
3535            ("H", Some(either), boom(), 1, None, true),
3536            ("I", Some(either), timeout(), 1, Some(1), true),
3537        ];
3538
3539        for (case, predicate, err, attempt, expected, called) in cases {
3540            let calls = Arc::new(AtomicU32::new(0));
3541            let mut builder = RetryStrategyConfig::builder()
3542                .max_attempts(3)
3543                .initial_delay(Duration::from_secs(1))
3544                .jitter(JitterStrategy::None);
3545            if let Some(predicate) = predicate {
3546                let calls = Arc::clone(&calls);
3547                builder = builder.retry_if(move |err| {
3548                    calls.fetch_add(1, Ordering::SeqCst);
3549                    predicate(err)
3550                });
3551            }
3552            let config = builder.build();
3553
3554            let decision = config.decide(&err, attempt);
3555            let expected = expected.map_or(RetryDecision::Stop, |secs| {
3556                RetryDecision::retry(Duration::from_secs(secs))
3557            });
3558            assert_eq!(decision, expected, "case {case}");
3559            assert_eq!(
3560                calls.load(Ordering::SeqCst),
3561                u32::from(called),
3562                "case {case}: predicate calls"
3563            );
3564        }
3565    }
3566
3567    /// The predicate sees the strategy's error kind, so a filter can tell
3568    /// an interrupted at-most-once attempt from a body error.
3569    #[test]
3570    fn retry_if_sees_the_error_kind() {
3571        let config = RetryStrategyConfig::builder()
3572            .max_attempts(3)
3573            .initial_delay(Duration::from_secs(1))
3574            .jitter(JitterStrategy::None)
3575            .retry_if(|err| matches!(err.kind(), StepErrorKind::ExecutionFailed))
3576            .build();
3577        let interrupted = StepError::new(StepErrorKind::Interrupted, None);
3578        assert_eq!(config.decide(&interrupted, 1), RetryDecision::Stop);
3579        assert_eq!(
3580            config.decide(&sample_error(), 1),
3581            RetryDecision::retry(Duration::from_secs(1))
3582        );
3583    }
3584
3585    /// `into_retry_strategy` passes the error to the filter.
3586    #[test]
3587    fn into_retry_strategy_applies_retry_if() {
3588        let strategy = RetryStrategyConfig::builder()
3589            .retry_if(|_| false)
3590            .build()
3591            .into_retry_strategy();
3592        assert_eq!(strategy(&sample_error(), 1), RetryDecision::Stop);
3593    }
3594}
3595
3596#[cfg(test)]
3597mod coordinator_tests {
3598    use crate::driver::InvocationOutcome;
3599    use crate::driver::test_support::{
3600        RefreshClient, RefreshRequest, assert_near, drive_handler, named, operation_like,
3601        poll_once, refresh_ctx, wire_err,
3602    };
3603    use aws_sdk_lambda::types::{
3604        ErrorObject, Operation, OperationAction, OperationStatus, OperationType, OperationUpdate,
3605        StepDetails,
3606    };
3607    use std::sync::atomic::{AtomicUsize, Ordering};
3608    use std::sync::{Arc, Mutex};
3609    use std::time::Duration;
3610
3611    fn secs(n: u64) -> Duration {
3612        Duration::from_secs(n)
3613    }
3614
3615    /// A step record of `update` with `status` and `attempt`, and, when
3616    /// `next_in` is `Some`, a next attempt `next_in` from now.
3617    fn step_record(
3618        req: &RefreshRequest<'_>,
3619        update: &OperationUpdate,
3620        status: OperationStatus,
3621        next_in: Option<Duration>,
3622    ) -> Operation {
3623        let mut details = StepDetails::builder().attempt(1);
3624        if let Some(next_in) = next_in {
3625            details = details.next_attempt_timestamp(req.timestamp_in(next_in));
3626        }
3627        operation_like(update, status)
3628            .step_details(details.build())
3629            .build()
3630            .expect("all required fields set")
3631    }
3632
3633    /// Answers a RETRY of `name` with `PENDING` and a next attempt 1 s
3634    /// ahead, and the first refresh with `name` `READY`.
3635    fn retry_client(name: &'static str) -> RefreshClient {
3636        RefreshClient::new(move |req| {
3637            let mut ops = Vec::new();
3638            if let Some(retry) = named(req.updates, name, &OperationAction::Retry) {
3639                ops.push(step_record(
3640                    req,
3641                    retry,
3642                    OperationStatus::Pending,
3643                    Some(secs(1)),
3644                ));
3645            }
3646            if req.empty_number == 1
3647                && let Some(retry) = named(req.history, name, &OperationAction::Retry)
3648            {
3649                ops.push(step_record(req, retry, OperationStatus::Ready, None));
3650            }
3651            Ok(ops)
3652        })
3653    }
3654
3655    /// Retries attempt 1 after 1 s and stops at attempt 2.
3656    fn retry_once(_err: &crate::StepError, attempt: u32) -> crate::RetryDecision {
3657        if attempt < 2 {
3658            crate::RetryDecision::retry(secs(1))
3659        } else {
3660            crate::RetryDecision::Stop
3661        }
3662    }
3663
3664    /// A step that sleeps 3 s of tokio time.
3665    async fn sleeping_step(ctx: &crate::DurableContext) -> Result<u32, crate::error::WireError> {
3666        ctx.step(|_| async {
3667            tokio::time::sleep(secs(3)).await;
3668            Ok(1_u32)
3669        })
3670        .name("s")
3671        .await
3672        .map_err(wire_err)
3673    }
3674
3675    #[tokio::test(start_paused = true)]
3676    async fn step_retry_runs_next_attempt_in_invocation() {
3677        let client = Arc::new(retry_client("flaky"));
3678        let ctx = refresh_ctx(&client);
3679        let start = tokio::time::Instant::now();
3680        let attempts = Arc::new(Mutex::new(Vec::new()));
3681        let body_attempts = Arc::clone(&attempts);
3682        let outcome = drive_handler(ctx, None, move |ctx| async move {
3683            let mut flaky = std::pin::pin!(
3684                ctx.step(move |step_ctx| {
3685                    let attempts = Arc::clone(&body_attempts);
3686                    async move {
3687                        attempts
3688                            .lock()
3689                            .expect("lock")
3690                            .push((step_ctx.attempt(), start.elapsed()));
3691                        if step_ctx.attempt() < 2 {
3692                            return Err::<u32, crate::BoxError>("transient".into());
3693                        }
3694                        Ok(7)
3695                    }
3696                })
3697                .name("flaky")
3698                .retry_strategy(retry_once)
3699                .spawn()
3700            );
3701            poll_once(flaky.as_mut()).await;
3702            sleeping_step(&ctx).await?;
3703            let value = flaky.await.map_err(wire_err)?;
3704            Ok(value.to_string())
3705        })
3706        .await
3707        .expect("no panic");
3708        assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3709        let attempts = attempts.lock().expect("lock").clone();
3710        let numbers: Vec<u32> = attempts.iter().map(|(n, _)| *n).collect();
3711        assert_eq!(numbers, vec![1, 2]);
3712        assert_near(attempts.get(1).expect("attempt 2 ran").1, secs(1));
3713    }
3714
3715    #[tokio::test(start_paused = true)]
3716    async fn step_retry_in_invocation_fires_start() {
3717        use crate::plugin::AttemptOutcome::{Failed, Succeeded};
3718        let client = Arc::new(retry_client("flaky"));
3719        let hooks = crate::plugin::test_support::OpHooks::default();
3720        let ctx = crate::DurableContext::new_root_with_client_and_defaults(
3721            "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
3722            lambda_runtime::Context::default(),
3723            Arc::new(crate::engine::CheckpointLog::empty()),
3724            Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
3725            "t0".to_owned(),
3726            Duration::ZERO,
3727            0,
3728            Some(hooks.plugins()),
3729        );
3730        let outcome = drive_handler(ctx, None, move |ctx| async move {
3731            let mut flaky = std::pin::pin!(
3732                ctx.step(|step_ctx| async move {
3733                    if step_ctx.attempt() < 2 {
3734                        return Err::<u32, crate::BoxError>("transient".into());
3735                    }
3736                    Ok(7)
3737                })
3738                .name("flaky")
3739                .retry_strategy(retry_once)
3740                .spawn()
3741            );
3742            poll_once(flaky.as_mut()).await;
3743            sleeping_step(&ctx).await?;
3744            let value = flaky.await.map_err(wire_err)?;
3745            Ok(value.to_string())
3746        })
3747        .await
3748        .expect("no panic");
3749        assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3750        let records = hooks.of("flaky");
3751        let shapes: Vec<_> = records
3752            .iter()
3753            .map(crate::plugin::test_support::OpRecord::shape)
3754            .collect();
3755        assert_eq!(
3756            shapes,
3757            [
3758                ("start", false, None, None),
3759                ("attempt_start", false, Some(1), None),
3760                ("attempt_end", false, Some(1), Some(Failed)),
3761                ("start", true, None, None),
3762                ("attempt_start", false, Some(2), None),
3763                ("attempt_end", false, Some(2), Some(Succeeded)),
3764                ("end", false, None, None),
3765            ]
3766        );
3767        let error = records.get(2).and_then(|r| r.error.as_deref());
3768        assert_eq!(error, Some("transient"));
3769        // The client returns no record for the SUCCEED, so the result is
3770        // the one the SDK wrote.
3771        let result = records.get(5).and_then(|r| r.result.as_deref());
3772        assert_eq!(result, Some("7"));
3773        let statuses: Vec<_> = records.iter().map(|r| r.status).collect();
3774        assert_eq!(
3775            statuses,
3776            [
3777                Some(crate::OperationStatus::Started),
3778                Some(crate::OperationStatus::Started),
3779                Some(crate::OperationStatus::Pending),
3780                Some(crate::OperationStatus::Started),
3781                Some(crate::OperationStatus::Started),
3782                Some(crate::OperationStatus::Succeeded),
3783                Some(crate::OperationStatus::Succeeded),
3784            ]
3785        );
3786    }
3787
3788    /// Rule 7.3: `on_attempt_start`, the user-function hooks and the
3789    /// wrap of attempt 2 share one info: `Started`, attempt 2, not a
3790    /// replay, with the attempt's start timestamp. `on_operation_start`
3791    /// of attempt 2 is a replay.
3792    #[tokio::test(start_paused = true)]
3793    async fn attempt_info_shared_by_hooks() {
3794        let client = Arc::new(retry_client("flaky"));
3795        let hooks = crate::plugin::recorder::UfHooks::default();
3796        let ctx = crate::DurableContext::new_root_with_client_and_defaults(
3797            "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
3798            lambda_runtime::Context::default(),
3799            Arc::new(crate::engine::CheckpointLog::empty()),
3800            Arc::clone(&client) as Arc<dyn crate::client::ExecutionClient>,
3801            "t0".to_owned(),
3802            Duration::ZERO,
3803            0,
3804            Some(hooks.plugins()),
3805        );
3806        let outcome = drive_handler(ctx, None, move |ctx| async move {
3807            let mut flaky = std::pin::pin!(
3808                ctx.step(|step_ctx| async move {
3809                    if step_ctx.attempt() < 2 {
3810                        return Err::<u32, crate::BoxError>("transient".into());
3811                    }
3812                    Ok(7)
3813                })
3814                .name("flaky")
3815                .retry_strategy(retry_once)
3816                .spawn()
3817            );
3818            poll_once(flaky.as_mut()).await;
3819            sleeping_step(&ctx).await?;
3820            let value = flaky.await.map_err(wire_err)?;
3821            Ok(value.to_string())
3822        })
3823        .await
3824        .expect("no panic");
3825        assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3826        let records = hooks.of("flaky");
3827        let starts: Vec<_> = records.iter().filter(|r| r.hook == "start").collect();
3828        assert_eq!(starts.len(), 2, "{records:?}");
3829        assert!(starts.get(1).is_some_and(|r| r.replay), "{starts:?}");
3830        let second = records
3831            .iter()
3832            .position(|r| r.hook == "attempt_start" && r.attempt == Some(2))
3833            .expect("attempt 2 started");
3834        let shared: Vec<_> = records
3835            .get(second..)
3836            .unwrap_or_default()
3837            .iter()
3838            .filter(|r| {
3839                matches!(
3840                    r.hook,
3841                    "attempt_start" | "uf_start" | "wrap_enter" | "wrap_exit" | "uf_end"
3842                )
3843            })
3844            .collect();
3845        let hook_names: Vec<_> = shared.iter().map(|r| r.hook).collect();
3846        assert_eq!(
3847            hook_names,
3848            [
3849                "attempt_start",
3850                "uf_start",
3851                "wrap_enter",
3852                "wrap_exit",
3853                "uf_end"
3854            ]
3855        );
3856        let first = shared.first().expect("attempt_start");
3857        for record in &shared {
3858            assert_eq!(
3859                record.info_addr, first.info_addr,
3860                "{} shares the info",
3861                record.hook
3862            );
3863            assert_eq!(record.status, Some(crate::OperationStatus::Started));
3864            assert_eq!(record.info_attempt, Some(2));
3865            assert!(!record.replay, "{}", record.hook);
3866            assert!(record.has_start_timestamp, "{}", record.hook);
3867        }
3868    }
3869
3870    #[tokio::test(start_paused = true)]
3871    async fn each_attempt_starts_from_original_captures() {
3872        let client = Arc::new(retry_client("vec"));
3873        let ctx = refresh_ctx(&client);
3874        let outcome = drive_handler(ctx, None, |ctx| async move {
3875            let v = vec![0_u32];
3876            let mut step = std::pin::pin!(
3877                ctx.step(move |step_ctx| async move {
3878                    let mut v = v;
3879                    v.push(step_ctx.attempt());
3880                    if step_ctx.attempt() < 2 {
3881                        return Err::<Vec<u32>, crate::BoxError>("transient".into());
3882                    }
3883                    Ok(v)
3884                })
3885                .name("vec")
3886                .retry_strategy(retry_once)
3887                .spawn()
3888            );
3889            poll_once(step.as_mut()).await;
3890            sleeping_step(&ctx).await?;
3891            let v = step.await.map_err(wire_err)?;
3892            Ok(format!("{v:?}"))
3893        })
3894        .await
3895        .expect("no panic");
3896        assert_eq!(outcome, InvocationOutcome::Complete("[0, 2]".to_owned()));
3897    }
3898
3899    #[tokio::test(start_paused = true)]
3900    async fn arc_capture_is_shared_across_attempts_in_invocation() {
3901        let client = Arc::new(retry_client("arc"));
3902        let ctx = refresh_ctx(&client);
3903        let counter = Arc::new(AtomicUsize::new(0));
3904        let handler_counter = Arc::clone(&counter);
3905        let outcome = drive_handler(ctx, None, move |ctx| async move {
3906            let mut step = std::pin::pin!(
3907                ctx.step(move |step_ctx| async move {
3908                    handler_counter.fetch_add(1, Ordering::SeqCst);
3909                    if step_ctx.attempt() < 2 {
3910                        return Err::<u32, crate::BoxError>("transient".into());
3911                    }
3912                    Ok(1)
3913                })
3914                .name("arc")
3915                .retry_strategy(retry_once)
3916                .spawn()
3917            );
3918            poll_once(step.as_mut()).await;
3919            sleeping_step(&ctx).await?;
3920            step.await.map_err(wire_err)?;
3921            Ok("done".to_owned())
3922        })
3923        .await
3924        .expect("no panic");
3925        assert_eq!(outcome, InvocationOutcome::Complete("done".to_owned()));
3926        assert_eq!(counter.load(Ordering::SeqCst), 2);
3927    }
3928
3929    #[tokio::test(start_paused = true)]
3930    async fn lone_step_retry_suspends() {
3931        let client = Arc::new(retry_client("lone"));
3932        let ctx = refresh_ctx(&client);
3933        let runs = Arc::new(AtomicUsize::new(0));
3934        let body_runs = Arc::clone(&runs);
3935        let outcome = drive_handler(ctx, None, move |ctx| async move {
3936            ctx.step(move |_| async move {
3937                body_runs.fetch_add(1, Ordering::SeqCst);
3938                Err::<u32, crate::BoxError>("transient".into())
3939            })
3940            .name("lone")
3941            .retry_strategy(retry_once)
3942            .await
3943            .map_err(wire_err)?;
3944            Ok("done".to_owned())
3945        })
3946        .await
3947        .expect("no panic");
3948        assert_eq!(outcome, InvocationOutcome::Pending);
3949        assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
3950        assert!(client.empty_calls().is_empty(), "no refresh was sent");
3951    }
3952
3953    #[tokio::test(start_paused = true)]
3954    async fn replayed_pending_step_runs_next_attempt_in_invocation() {
3955        let client = Arc::new(RefreshClient::new(|req| {
3956            let mut ops = Vec::new();
3957            if req.empty_number == 1 {
3958                ops.push(
3959                    operation_like(&flaky_update(), OperationStatus::Ready)
3960                        .step_details(StepDetails::builder().attempt(1).build())
3961                        .build()
3962                        .expect("all required fields set"),
3963                );
3964            }
3965            Ok(ops)
3966        }));
3967        let ctx = pending_flaky_ctx(&client);
3968        let attempts = Arc::new(Mutex::new(Vec::new()));
3969        let body_attempts = Arc::clone(&attempts);
3970        let outcome = drive_handler(ctx, None, move |ctx| async move {
3971            let mut flaky = std::pin::pin!(
3972                ctx.step(move |step_ctx| {
3973                    let attempts = Arc::clone(&body_attempts);
3974                    async move {
3975                        attempts.lock().expect("lock").push(step_ctx.attempt());
3976                        Ok::<u32, crate::BoxError>(7)
3977                    }
3978                })
3979                .name("flaky")
3980                .retry_strategy(retry_once)
3981                .spawn()
3982            );
3983            poll_once(flaky.as_mut()).await;
3984            sleeping_step(&ctx).await?;
3985            let value = flaky.await.map_err(wire_err)?;
3986            Ok(value.to_string())
3987        })
3988        .await
3989        .expect("no panic");
3990        assert_eq!(outcome, InvocationOutcome::Complete("7".to_owned()));
3991        assert_eq!(*attempts.lock().expect("lock"), vec![2]);
3992    }
3993
3994    /// The root context over a log that holds the step `flaky` `PENDING`
3995    /// after attempt 1, with its next attempt 1 s ahead.
3996    fn pending_flaky_ctx(client: &Arc<RefreshClient>) -> crate::DurableContext {
3997        let pending = operation_like(&flaky_update(), OperationStatus::Pending)
3998            .step_details(
3999                StepDetails::builder()
4000                    .attempt(1)
4001                    .next_attempt_timestamp(aws_smithy_types::DateTime::from(
4002                        std::time::SystemTime::now() + secs(1),
4003                    ))
4004                    .build(),
4005            )
4006            .build()
4007            .expect("all required fields set");
4008        crate::DurableContext::new_root_with_client(
4009            "arn:aws:lambda:us-east-1:1:function:test".to_owned(),
4010            lambda_runtime::Context::default(),
4011            Arc::new(crate::client::operations_to_checkpoint_log(&[pending])),
4012            Arc::clone(client) as Arc<dyn crate::client::ExecutionClient>,
4013            "t0".to_owned(),
4014        )
4015    }
4016
4017    /// A waiting step whose record turns terminal returns the recorded
4018    /// outcome in the invocation, without running the body and without an
4019    /// `operation_replayed` event.
4020    #[tokio::test(start_paused = true)]
4021    async fn pending_step_returns_terminal_record_in_invocation() {
4022        let (events, _guard) = crate::driver::test_support::EventMessages::install();
4023        let client = Arc::new(RefreshClient::new(|req| {
4024            let mut ops = Vec::new();
4025            if req.empty_number == 1 {
4026                ops.push(
4027                    operation_like(&flaky_update(), OperationStatus::Succeeded)
4028                        .step_details(StepDetails::builder().attempt(2).result("5").build())
4029                        .build()
4030                        .expect("all required fields set"),
4031                );
4032            }
4033            Ok(ops)
4034        }));
4035        let ctx = pending_flaky_ctx(&client);
4036        let runs = Arc::new(AtomicUsize::new(0));
4037        let body_runs = Arc::clone(&runs);
4038        let outcome = drive_handler(ctx, None, move |ctx| async move {
4039            let mut flaky = std::pin::pin!(
4040                ctx.step(move |_| async move {
4041                    body_runs.fetch_add(1, Ordering::SeqCst);
4042                    Ok::<u32, crate::BoxError>(7)
4043                })
4044                .name("flaky")
4045                .spawn()
4046            );
4047            poll_once(flaky.as_mut()).await;
4048            sleeping_step(&ctx).await?;
4049            let value = flaky.await.map_err(wire_err)?;
4050            Ok(value.to_string())
4051        })
4052        .await
4053        .expect("no panic");
4054        assert_eq!(outcome, InvocationOutcome::Complete("5".to_owned()));
4055        assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
4056        assert_eq!(
4057            events.replayed(),
4058            0,
4059            "a resumed record emits no replay event"
4060        );
4061    }
4062
4063    /// A waiting step whose record turns FAILED returns the recorded error
4064    /// in the invocation, without running the body and without an
4065    /// `operation_replayed` event.
4066    #[tokio::test(start_paused = true)]
4067    async fn pending_step_returns_failed_record_in_invocation() {
4068        let (events, _guard) = crate::driver::test_support::EventMessages::install();
4069        let client = Arc::new(RefreshClient::new(|req| {
4070            let mut ops = Vec::new();
4071            if req.empty_number == 1 {
4072                ops.push(
4073                    operation_like(&flaky_update(), OperationStatus::Failed)
4074                        .step_details(
4075                            StepDetails::builder()
4076                                .attempt(2)
4077                                .error(
4078                                    ErrorObject::builder()
4079                                        .error_type("PaymentError")
4080                                        .error_message("declined")
4081                                        .build(),
4082                                )
4083                                .build(),
4084                        )
4085                        .build()
4086                        .expect("all required fields set"),
4087                );
4088            }
4089            Ok(ops)
4090        }));
4091        let ctx = pending_flaky_ctx(&client);
4092        let runs = Arc::new(AtomicUsize::new(0));
4093        let body_runs = Arc::clone(&runs);
4094        let outcome = drive_handler(ctx, None, move |ctx| async move {
4095            let mut flaky = std::pin::pin!(
4096                ctx.step(move |_| async move {
4097                    body_runs.fetch_add(1, Ordering::SeqCst);
4098                    Ok::<u32, crate::BoxError>(7)
4099                })
4100                .name("flaky")
4101                .spawn()
4102            );
4103            poll_once(flaky.as_mut()).await;
4104            sleeping_step(&ctx).await?;
4105            let err = flaky.await.expect_err("the recorded failure returns");
4106            let error_type = err
4107                .wire()
4108                .and_then(crate::WireError::error_type)
4109                .unwrap_or_default()
4110                .to_owned();
4111            Ok(error_type)
4112        })
4113        .await
4114        .expect("no panic");
4115        assert_eq!(
4116            outcome,
4117            InvocationOutcome::Complete("PaymentError".to_owned())
4118        );
4119        assert_eq!(runs.load(Ordering::SeqCst), 0, "the body never ran");
4120        assert_eq!(
4121            events.replayed(),
4122            0,
4123            "a resumed record emits no replay event"
4124        );
4125    }
4126
4127    /// The identity of the first root step, named `flaky`, as an update.
4128    fn flaky_update() -> OperationUpdate {
4129        OperationUpdate::builder()
4130            .id(crate::engine::compute_wire_id_public("1"))
4131            .r#type(OperationType::Step)
4132            .sub_type(super::STEP_SUB_TYPE)
4133            .name("flaky")
4134            .action(OperationAction::Start)
4135            .build()
4136            .expect("all required fields set")
4137    }
4138}
4139
4140/// When an attempt's body runs relative to its `START` write. Every test
4141/// runs under paused tokio time, with a client that answers each
4142/// checkpoint call 500 ms after it receives it.
4143#[cfg(test)]
4144mod start_write_tests {
4145    use crate::StepSemantics;
4146    use crate::driver::InvocationOutcome;
4147    use crate::driver::test_support::{LatencyClient, drive_handler, latency_ctx, wire_err};
4148    use aws_sdk_lambda::types::OperationAction;
4149    use std::sync::atomic::{AtomicUsize, Ordering};
4150    use std::sync::{Arc, Mutex};
4151    use std::time::Duration;
4152
4153    const LATENCY: Duration = Duration::from_millis(500);
4154
4155    fn ms(n: u64) -> Duration {
4156        Duration::from_millis(n)
4157    }
4158
4159    /// Runs, on a client with [`LATENCY`] and `buffer_window`, a handler
4160    /// whose step records the tokio time its body starts and returns `1`.
4161    /// Returns the client, the time the body started, and the handler's
4162    /// outcome: the step's value and the time the step returned.
4163    async fn timed_step(
4164        buffer_window: Duration,
4165        semantics: StepSemantics,
4166    ) -> (Arc<LatencyClient>, Duration, InvocationOutcome) {
4167        let client = Arc::new(LatencyClient::new(LATENCY));
4168        let ctx = latency_ctx(&client, buffer_window);
4169        let started = Arc::new(Mutex::new(None));
4170        let body_started = Arc::clone(&started);
4171        let clock = Arc::clone(&client);
4172        let outcome = drive_handler(ctx, None, move |ctx| async move {
4173            let body_clock = Arc::clone(&clock);
4174            let value = ctx
4175                .step(move |_| {
4176                    let started = Arc::clone(&body_started);
4177                    let clock = Arc::clone(&body_clock);
4178                    async move {
4179                        *started.lock().expect("lock") = Some(clock.elapsed());
4180                        Ok(1_u32)
4181                    }
4182                })
4183                .name("s")
4184                .semantics(semantics)
4185                .await
4186                .map_err(wire_err)?;
4187            Ok(format!("{value}@{}", clock.elapsed().as_millis()))
4188        })
4189        .await
4190        .expect("no panic");
4191        let started = started.lock().expect("lock").expect("the body ran");
4192        (client, started, outcome)
4193    }
4194
4195    #[tokio::test(start_paused = true)]
4196    async fn at_least_once_body_starts_before_start_response() {
4197        let (client, started, outcome) =
4198            timed_step(Duration::ZERO, StepSemantics::AtLeastOncePerRetry).await;
4199        assert_eq!(started, ms(0), "the body starts before START is answered");
4200        let start = client.request_with(&OperationAction::Start).expect("START");
4201        let succeed = client
4202            .request_with(&OperationAction::Succeed)
4203            .expect("SUCCEED");
4204        assert_eq!(start.received, ms(0));
4205        assert_eq!(succeed.received, ms(500), "SUCCEED waits for START");
4206        assert_eq!(outcome, InvocationOutcome::Complete("1@1000".to_owned()));
4207    }
4208
4209    #[tokio::test(start_paused = true)]
4210    async fn buffered_start_precedes_outcome() {
4211        let (client, started, outcome) =
4212            timed_step(ms(50), StepSemantics::AtLeastOncePerRetry).await;
4213        assert_eq!(started, ms(0), "the body starts before START is answered");
4214        let requests = client.requests();
4215        let first = requests.first().expect("a request");
4216        let actions: Vec<&OperationAction> = first
4217            .updates
4218            .iter()
4219            .map(aws_sdk_lambda::types::OperationUpdate::action)
4220            .collect();
4221        assert_eq!(
4222            actions,
4223            vec![&OperationAction::Start],
4224            "the first request carries only START"
4225        );
4226        assert!(
4227            requests.iter().skip(1).any(|r| r
4228                .updates
4229                .iter()
4230                .any(|u| u.action() == &OperationAction::Succeed)),
4231            "SUCCEED is in a later request"
4232        );
4233        assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
4234    }
4235
4236    #[tokio::test(start_paused = true)]
4237    async fn at_most_once_body_waits_for_start_response() {
4238        let (_client, started, outcome) =
4239            timed_step(Duration::ZERO, StepSemantics::AtMostOncePerRetry).await;
4240        assert_eq!(started, ms(500), "the body starts after START is answered");
4241        assert!(matches!(outcome, InvocationOutcome::Complete(ref v) if v.starts_with("1@")));
4242    }
4243
4244    /// Asserts that a drive whose `START` write failed failed the
4245    /// execution with the checkpoint error's type, and wrote no outcome.
4246    fn assert_failed_without_outcome(client: &LatencyClient, outcome: &InvocationOutcome) {
4247        let InvocationOutcome::Failed { error } = outcome else {
4248            unreachable!("expected a failed execution, got {outcome:?}");
4249        };
4250        assert_eq!(
4251            error.error_type(),
4252            Some(crate::error::CHECKPOINT_EXECUTION_FAILED_ERROR_TYPE)
4253        );
4254        assert!(client.request_with(&OperationAction::Start).is_some());
4255        assert!(!client.received_outcome(), "no outcome update was sent");
4256    }
4257
4258    #[tokio::test(start_paused = true)]
4259    async fn failed_start_write_writes_no_outcome() {
4260        // (a) A step.
4261        let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
4262        let runs = Arc::new(AtomicUsize::new(0));
4263        let body_runs = Arc::clone(&runs);
4264        let outcome = drive_handler(
4265            latency_ctx(&client, Duration::ZERO),
4266            None,
4267            move |ctx| async move {
4268                ctx.step(move |_| async move {
4269                    body_runs.fetch_add(1, Ordering::SeqCst);
4270                    Ok(1_u32)
4271                })
4272                .name("s")
4273                .await
4274                .map_err(wire_err)?;
4275                Ok("done".to_owned())
4276            },
4277        )
4278        .await
4279        .expect("no panic");
4280        assert_eq!(runs.load(Ordering::SeqCst), 1, "the body ran once");
4281        assert_failed_without_outcome(&client, &outcome);
4282
4283        // (b) A condition check.
4284        let client = Arc::new(LatencyClient::new(LATENCY).failing_start());
4285        let checks = Arc::new(AtomicUsize::new(0));
4286        let check_runs = Arc::clone(&checks);
4287        let outcome = drive_handler(
4288            latency_ctx(&client, Duration::ZERO),
4289            None,
4290            move |ctx| async move {
4291                ctx.wait_for_condition(
4292                    move |_, state: u32| {
4293                        let checks = Arc::clone(&check_runs);
4294                        async move {
4295                            checks.fetch_add(1, Ordering::SeqCst);
4296                            Ok(state + 1)
4297                        }
4298                    },
4299                    0,
4300                )
4301                .wait_strategy_fn(|_state, _attempt| {
4302                    crate::builders::wait_for_condition::WaitDecision::complete()
4303                })
4304                .name("c")
4305                .await
4306                .map_err(wire_err)?;
4307                Ok("done".to_owned())
4308            },
4309        )
4310        .await
4311        .expect("no panic");
4312        assert_eq!(checks.load(Ordering::SeqCst), 1, "the check ran once");
4313        assert_failed_without_outcome(&client, &outcome);
4314    }
4315
4316    /// Asserts that a drive whose body or check panicked with `"boom"`
4317    /// before its `START` was answered kept the `START` write: the write
4318    /// was answered, no outcome was sent, the record is `STARTED`, and the
4319    /// panic left the drive only after the answer.
4320    fn assert_start_kept(
4321        client: &LatencyClient,
4322        ctx: &crate::DurableContext,
4323        result: Result<InvocationOutcome, Box<dyn std::any::Any + Send + 'static>>,
4324        returned: Duration,
4325    ) {
4326        let start = client.request_with(&OperationAction::Start).expect("START");
4327        let answered = start.answered.expect("START was answered");
4328        assert_eq!(answered, start.received + LATENCY);
4329        assert!(!client.received_outcome(), "no outcome update was sent");
4330        let wire_id = start.updates.first().expect("an update").id();
4331        assert_eq!(crate::engine::compute_wire_id_public("1"), wire_id);
4332        let view = ctx.checkpoint_status_view("1").expect("a record");
4333        assert_eq!(view.status, crate::engine::CheckpointStatus::Started);
4334        let payload = result.expect_err("the panic leaves the drive");
4335        assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom"));
4336        assert!(
4337            returned >= answered,
4338            "the drive returned at {returned:?}, before {answered:?}"
4339        );
4340    }
4341
4342    /// Drives, on a client with [`LATENCY`] and `buffer_window`, a step
4343    /// whose body panics with `"boom"` at once, and asserts the `START`
4344    /// write was kept.
4345    async fn panicking_step(buffer_window: Duration) {
4346        let client = Arc::new(LatencyClient::new(LATENCY));
4347        let ctx = latency_ctx(&client, buffer_window);
4348        let clock = Arc::clone(&client);
4349        let result = drive_handler(ctx.clone(), None, |ctx| async move {
4350            #[expect(clippy::panic)] // reason: the scenario is a panicking body
4351            let (): () = ctx
4352                .step(move |_| async move {
4353                    assert_eq!(
4354                        clock.elapsed(),
4355                        ms(0),
4356                        "the body runs before START is answered"
4357                    );
4358                    panic!("boom")
4359                })
4360                .name("s")
4361                .await
4362                .map_err(wire_err)?;
4363            Ok("done".to_owned())
4364        })
4365        .await;
4366        assert_start_kept(&client, &ctx, result, client.elapsed());
4367    }
4368
4369    #[tokio::test(start_paused = true)]
4370    async fn body_panic_before_start_response_keeps_start_write() {
4371        // (a) No coalescing window.
4372        panicking_step(Duration::ZERO).await;
4373        // (b) A 50 ms coalescing window.
4374        panicking_step(ms(50)).await;
4375
4376        // (c) A condition check, no coalescing window.
4377        let client = Arc::new(LatencyClient::new(LATENCY));
4378        let ctx = latency_ctx(&client, Duration::ZERO);
4379        let clock = Arc::clone(&client);
4380        let result = drive_handler(ctx.clone(), None, |ctx| async move {
4381            #[expect(clippy::panic)] // reason: the scenario is a panicking check
4382            ctx.wait_for_condition(
4383                move |_, _state: u32| {
4384                    let clock = Arc::clone(&clock);
4385                    async move {
4386                        assert_eq!(
4387                            clock.elapsed(),
4388                            ms(0),
4389                            "the check runs before START is answered"
4390                        );
4391                        panic!("boom")
4392                    }
4393                },
4394                0,
4395            )
4396            .wait_strategy_fn(|_state, _attempt| {
4397                crate::builders::wait_for_condition::WaitDecision::complete()
4398            })
4399            .name("c")
4400            .await
4401            .map_err(wire_err)?;
4402            Ok("done".to_owned())
4403        })
4404        .await;
4405        assert_start_kept(&client, &ctx, result, client.elapsed());
4406    }
4407}