aws-durable-execution-sdk 1.0.0

Rust SDK for AWS Lambda Durable Functions
Documentation
//! Block-level retry: run a closure against a child context and apply a
//! retry strategy to the closure's OVERALL outcome, so a multi-operation
//! block retries as a unit.
//!
//! Structure on the wire: everything is expressed through the existing,
//! backend-proven child-context and wait protocols; no new operation types
//! are introduced:
//!
//! - The `with_retry` operation itself is a child context
//!   (`OperationType::Context`) whose body is the retry loop. Its sub-type
//!   is the one the caller sets with `WithRetryBuilder::sub_type`, and
//!   `RunInChildContext` when the caller sets none. With
//!   `WithRetryBuilder::virtual_context`, the block is a virtual child
//!   context: the loop runs in its own namespace but the block writes no
//!   record, and a block named `b` names its attempts `b-attempt-N`.
//! - Each ATTEMPT is a nested child context under it (named `attempt-N`,
//!   sub-type `RunInChildContext` whatever the outer sub-type is),
//!   which is what gives every attempt a fresh operation namespace: attempt
//!   1's operations live under `<id>-1-*`, attempt 2's under `<id>-3-*`,
//!   and so on. A failed attempt's recorded operations can therefore never
//!   be confused with (or replayed into) the next attempt's.
//! - The delay after failed attempt N is a durable wait, named
//!   `{name}-backoff-N` when the block has a non-empty name and unnamed
//!   otherwise, as in the JS SDK. So the backend owns the retry timer, and
//!   the block waits between attempts exactly as a retrying step does: the
//!   invocation suspends during the wait only when no other SDK work can
//!   make progress.
//!
//! Retry-state checkpointing: the loop carries no in-process state that
//! matters across a suspension. The current attempt number is re-derived on
//! every (re)entry purely from recorded checkpoint results: a finished
//! attempt replays its frozen outcome (a failed attempt replays its `Fail`
//! record without re-running its body), the strategy decision is a
//! deterministic function of that recorded error and the attempt number,
//! and the inter-attempt wait is itself a checkpointed operation. The retry
//! state therefore survives suspension by construction.

use std::future::{Future, IntoFuture};
use std::sync::Arc;

use crate::context::DurableContext;
use crate::error::{ChildFnError, OperationError, StepError};
use crate::future::DurableFuture;
use crate::serdes::Serdes;
use crate::{BoxError, RetryDecision, RetryStrategy};

/// Whether a `with_retry` block writes a child context record of its own.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) enum RetryWrap {
    /// The block is a child context, and its attempts are `attempt-N`
    /// children inside it.
    #[default]
    Child,
    /// The block is a virtual child context: it claims an ID and its
    /// attempts and waits are created in its own namespace, but it writes
    /// no record, so they name the caller's parent as their parent.
    Virtual,
}

/// Why the retry loop ended without a value.
#[derive(Debug)]
pub(crate) enum LoopFailure {
    /// The strategy stopped: the last attempt's error, unchanged.
    Exhausted(OperationError),
    /// The wait between two attempts failed: the wait's error.
    Wait(OperationError),
}

impl LoopFailure {
    /// The error a block with its own child context records: the last
    /// attempt's error as the child failure's direct source, or the failed
    /// wait under `with_retry delay wait failed`.
    pub(crate) fn into_child_error(self) -> ChildFnError {
        match self {
            Self::Exhausted(last) => ChildFnError::new(last),
            Self::Wait(wait_err) => ChildFnError::new(crate::error::ContextualError::source_from(
                "with_retry delay wait failed",
                Box::new(wait_err) as crate::error::Source,
            )),
        }
    }
}

/// The name of attempt `attempt`: `{block}-attempt-N` for a virtual block
/// with a non-empty name, `attempt-N` otherwise.
pub(crate) fn attempt_name(block: Option<&str>, wrap: RetryWrap, attempt: u32) -> String {
    match (wrap, block) {
        (RetryWrap::Virtual, Some(block)) if !block.is_empty() => {
            format!("{block}-attempt-{attempt}")
        }
        _ => format!("attempt-{attempt}"),
    }
}

/// Creates the future of attempt `attempt` as a nested child context of
/// `outer`, which claims its operation ID now.
///
/// The closure is shared as `Arc<F>`, because the block runs it once per
/// attempt, and receives the attempt number with the attempt's context.
/// Each attempt round-trips `O` through the builder's serdes, shared
/// through the forwarding `impl Serdes<T> for Arc<S>`.
pub(crate) fn start_attempt<O, F, Fut, S>(
    outer: &DurableContext,
    name: String,
    attempt: u32,
    closure: &Arc<F>,
    serdes: &Arc<S>,
) -> DurableFuture<O>
where
    O: Send + 'static,
    F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
    Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
    S: Serdes<O>,
{
    let f = Arc::clone(closure);
    let n = attempt;
    outer
        .run_in_child_context(move |attempt_ctx| f(attempt_ctx, n))
        .without_user_function()
        .name(name)
        .serdes(Arc::clone(serdes))
        .into_future()
}

/// Runs the retry loop in `outer`.
///
/// `outer` is the block's own child context, virtual for a virtual block.
/// The loop creates one nested child context per attempt plus one wait per
/// retry delay in it. Returns the first successful attempt's value, or the
/// [`LoopFailure`] that ended the loop.
///
/// The closure is generic (`Arc<F>` rather than a boxed `dyn Fn`): each
/// attempt produces a concrete future without its own box, so the single
/// erasure point stays at the enclosing builder's
/// [`DurableFuture`](crate::DurableFuture). `name` is the block's name; it
/// names the waits between attempts, and a virtual block's attempts.
pub(crate) async fn retry_loop<O, F, Fut, S>(
    outer: DurableContext,
    name: Option<String>,
    wrap: RetryWrap,
    closure: Arc<F>,
    strategy: Arc<RetryStrategy>,
    serdes: Arc<S>,
) -> Result<O, LoopFailure>
where
    O: Send + 'static,
    F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
    Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
    S: Serdes<O>,
{
    let mut attempt: u32 = 1;
    loop {
        // One nested child context per attempt: a fresh operation
        // namespace, so nothing recorded by a previous (failed) attempt
        // replays into this one. On replay, a finished attempt returns its
        // frozen outcome without calling the closure.
        let future = start_attempt(
            &outer,
            attempt_name(name.as_deref(), wrap, attempt),
            attempt,
            &closure,
            &serdes,
        );
        let err = match future.await {
            Ok(value) => return Ok(value),
            Err(err) => err,
        };

        // Present the failure to the strategy in the same shape a step's
        // strategy sees: the attempt's error is the step error's source.
        // The strategy judges an attempt's failure as a replay of the
        // attempt rebuilds it from its record, so it decides the same in
        // every invocation. An error without a record, such as the decode
        // failure of a succeeded attempt's result, reaches it unchanged.
        let (judged, kept) = match err.wire().zip(err.operation_id()) {
            Some((wire, id)) => (crate::child::replay_failure(wire.clone(), id), Some(err)),
            None => (err, None),
        };
        let step_err = StepError::execution_failed(judged);
        match strategy(&step_err, attempt) {
            RetryDecision::Retry { delay } => {
                // Durable wait between attempts: checkpointed, and the
                // backend owns the timer. `ctx.wait` rounds the delay up
                // to whole seconds with a one-second minimum. JS names
                // this wait `{name}-backoff-{attempt}` only when the
                // block's name is non-empty, so an unnamed block's waits
                // are unnamed.
                let wait = outer.wait(delay);
                let wait = match name.as_deref() {
                    Some(name) if !name.is_empty() => {
                        wait.name(format!("{name}-backoff-{attempt}"))
                    }
                    _ => wait,
                };
                wait.await.map_err(LoopFailure::Wait)?;
                attempt = attempt.saturating_add(1);
            }
            RetryDecision::Stop => {
                let last = kept.unwrap_or_else(|| last_error(step_err));
                return Err(LoopFailure::Exhausted(last));
            }
        }
    }
}

/// Takes the attempt's error back out of the step error the strategy saw.
#[expect(clippy::expect_used)]
// reason: `retry_loop` builds the step error with
// `StepError::execution_failed` of an `OperationError`, which is its source, so both the source and the downcast are always present.
fn last_error(step_err: StepError) -> OperationError {
    *step_err
        .into_source()
        .expect("the step error carries the attempt's error")
        .downcast::<OperationError>()
        .expect("the attempt's error is an OperationError")
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::error::{ChildContextError, ChildContextErrorKind, OperationErrorKind};

    fn op_error(kind: ChildContextErrorKind) -> OperationError {
        OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
            kind, None,
        )))
    }

    /// Exhaustion makes the last attempt's error the child failure's
    /// direct source; a failed wait goes under `with_retry delay wait
    /// failed`.
    #[test]
    fn loop_failure_maps_to_child_error() {
        let exhausted = LoopFailure::Exhausted(op_error(ChildContextErrorKind::ChildFailed))
            .into_child_error()
            .into_source();
        let last = exhausted
            .downcast_ref::<OperationError>()
            .expect("the attempt error is the direct source");
        assert!(matches!(last.kind(), OperationErrorKind::ChildContext(_)));

        let waited = LoopFailure::Wait(op_error(ChildContextErrorKind::Internal))
            .into_child_error()
            .into_source();
        assert_eq!(waited.to_string(), "with_retry delay wait failed");
        assert!(
            std::error::Error::source(&*waited)
                .and_then(|s| s.downcast_ref::<OperationError>())
                .is_some()
        );
    }

    /// A failed attempt reaches the strategy as
    /// `StepError::execution_failed` of the attempt's error.
    #[cfg(feature = "test-util")]
    #[tokio::test]
    async fn failed_with_retry_attempt_gives_strategy_step_error_execution_failed() {
        type Seen = Arc<std::sync::Mutex<Vec<(String, Option<String>)>>>;
        let seen: Seen = Arc::default();
        let returned: Arc<std::sync::Mutex<Option<String>>> = Arc::default();
        let (seen_in, returned_in) = (Arc::clone(&seen), Arc::clone(&returned));
        crate::test_util::LocalRunner::new()
            .run(
                move |(): (), ctx: DurableContext| {
                    let (seen, returned) = (Arc::clone(&seen_in), Arc::clone(&returned_in));
                    async move {
                        let err = ctx
                            .with_retry(|_child, _attempt| async {
                                Err::<u32, BoxError>("boom".into())
                            })
                            .name("block")
                            .retry_strategy(move |err, _attempt| {
                                let source =
                                    std::error::Error::source(err).map(ToString::to_string);
                                seen.lock()
                                    .expect("strategy log lock")
                                    .push((err.kind().to_string(), source));
                                RetryDecision::Stop
                            })
                            .await
                            .expect_err("the block fails");
                        // The block's child failure has the last attempt's
                        // error as its source.
                        *returned.lock().expect("returned lock") = std::error::Error::source(&err)
                            .and_then(std::error::Error::source)
                            .map(ToString::to_string);
                        Ok::<(), BoxError>(())
                    }
                },
                (),
            )
            .await
            .expect("the run finishes");

        let attempt_error = returned.lock().expect("returned lock").clone();
        assert!(
            attempt_error.is_some(),
            "the block returned its attempt's error"
        );
        let expected = StepError::execution_failed(attempt_error.clone().unwrap_or_default());
        assert_eq!(
            *seen.lock().expect("strategy log lock"),
            [(
                expected.kind().to_string(),
                std::error::Error::source(&expected).map(ToString::to_string),
            )]
        );
    }

    /// Only a virtual block with a non-empty name prefixes its attempts.
    #[test]
    fn attempt_names() {
        assert_eq!(attempt_name(Some("b"), RetryWrap::Child, 2), "attempt-2");
        assert_eq!(
            attempt_name(Some("b"), RetryWrap::Virtual, 2),
            "b-attempt-2"
        );
        assert_eq!(attempt_name(Some(""), RetryWrap::Virtual, 2), "attempt-2");
        assert_eq!(attempt_name(None, RetryWrap::Virtual, 2), "attempt-2");
    }
}