use std::future::{Future, IntoFuture};
use std::sync::Arc;
use crate::context::DurableContext;
use crate::error::{ChildFnError, OperationError, StepError};
use crate::future::DurableFuture;
use crate::serdes::Serdes;
use crate::{BoxError, RetryDecision, RetryStrategy};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) enum RetryWrap {
#[default]
Child,
Virtual,
}
#[derive(Debug)]
pub(crate) enum LoopFailure {
Exhausted(OperationError),
Wait(OperationError),
}
impl LoopFailure {
pub(crate) fn into_child_error(self) -> ChildFnError {
match self {
Self::Exhausted(last) => ChildFnError::new(last),
Self::Wait(wait_err) => ChildFnError::new(crate::error::ContextualError::source_from(
"with_retry delay wait failed",
Box::new(wait_err) as crate::error::Source,
)),
}
}
}
pub(crate) fn attempt_name(block: Option<&str>, wrap: RetryWrap, attempt: u32) -> String {
match (wrap, block) {
(RetryWrap::Virtual, Some(block)) if !block.is_empty() => {
format!("{block}-attempt-{attempt}")
}
_ => format!("attempt-{attempt}"),
}
}
pub(crate) fn start_attempt<O, F, Fut, S>(
outer: &DurableContext,
name: String,
attempt: u32,
closure: &Arc<F>,
serdes: &Arc<S>,
) -> DurableFuture<O>
where
O: Send + 'static,
F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
S: Serdes<O>,
{
let f = Arc::clone(closure);
let n = attempt;
outer
.run_in_child_context(move |attempt_ctx| f(attempt_ctx, n))
.without_user_function()
.name(name)
.serdes(Arc::clone(serdes))
.into_future()
}
pub(crate) async fn retry_loop<O, F, Fut, S>(
outer: DurableContext,
name: Option<String>,
wrap: RetryWrap,
closure: Arc<F>,
strategy: Arc<RetryStrategy>,
serdes: Arc<S>,
) -> Result<O, LoopFailure>
where
O: Send + 'static,
F: Fn(DurableContext, u32) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, BoxError>> + Send + 'static,
S: Serdes<O>,
{
let mut attempt: u32 = 1;
loop {
let future = start_attempt(
&outer,
attempt_name(name.as_deref(), wrap, attempt),
attempt,
&closure,
&serdes,
);
let err = match future.await {
Ok(value) => return Ok(value),
Err(err) => err,
};
let (judged, kept) = match err.wire().zip(err.operation_id()) {
Some((wire, id)) => (crate::child::replay_failure(wire.clone(), id), Some(err)),
None => (err, None),
};
let step_err = StepError::execution_failed(judged);
match strategy(&step_err, attempt) {
RetryDecision::Retry { delay } => {
let wait = outer.wait(delay);
let wait = match name.as_deref() {
Some(name) if !name.is_empty() => {
wait.name(format!("{name}-backoff-{attempt}"))
}
_ => wait,
};
wait.await.map_err(LoopFailure::Wait)?;
attempt = attempt.saturating_add(1);
}
RetryDecision::Stop => {
let last = kept.unwrap_or_else(|| last_error(step_err));
return Err(LoopFailure::Exhausted(last));
}
}
}
}
#[expect(clippy::expect_used)]
fn last_error(step_err: StepError) -> OperationError {
*step_err
.into_source()
.expect("the step error carries the attempt's error")
.downcast::<OperationError>()
.expect("the attempt's error is an OperationError")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::{ChildContextError, ChildContextErrorKind, OperationErrorKind};
fn op_error(kind: ChildContextErrorKind) -> OperationError {
OperationError::from_kind(OperationErrorKind::ChildContext(ChildContextError::new(
kind, None,
)))
}
#[test]
fn loop_failure_maps_to_child_error() {
let exhausted = LoopFailure::Exhausted(op_error(ChildContextErrorKind::ChildFailed))
.into_child_error()
.into_source();
let last = exhausted
.downcast_ref::<OperationError>()
.expect("the attempt error is the direct source");
assert!(matches!(last.kind(), OperationErrorKind::ChildContext(_)));
let waited = LoopFailure::Wait(op_error(ChildContextErrorKind::Internal))
.into_child_error()
.into_source();
assert_eq!(waited.to_string(), "with_retry delay wait failed");
assert!(
std::error::Error::source(&*waited)
.and_then(|s| s.downcast_ref::<OperationError>())
.is_some()
);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn failed_with_retry_attempt_gives_strategy_step_error_execution_failed() {
type Seen = Arc<std::sync::Mutex<Vec<(String, Option<String>)>>>;
let seen: Seen = Arc::default();
let returned: Arc<std::sync::Mutex<Option<String>>> = Arc::default();
let (seen_in, returned_in) = (Arc::clone(&seen), Arc::clone(&returned));
crate::test_util::LocalRunner::new()
.run(
move |(): (), ctx: DurableContext| {
let (seen, returned) = (Arc::clone(&seen_in), Arc::clone(&returned_in));
async move {
let err = ctx
.with_retry(|_child, _attempt| async {
Err::<u32, BoxError>("boom".into())
})
.name("block")
.retry_strategy(move |err, _attempt| {
let source =
std::error::Error::source(err).map(ToString::to_string);
seen.lock()
.expect("strategy log lock")
.push((err.kind().to_string(), source));
RetryDecision::Stop
})
.await
.expect_err("the block fails");
*returned.lock().expect("returned lock") = std::error::Error::source(&err)
.and_then(std::error::Error::source)
.map(ToString::to_string);
Ok::<(), BoxError>(())
}
},
(),
)
.await
.expect("the run finishes");
let attempt_error = returned.lock().expect("returned lock").clone();
assert!(
attempt_error.is_some(),
"the block returned its attempt's error"
);
let expected = StepError::execution_failed(attempt_error.clone().unwrap_or_default());
assert_eq!(
*seen.lock().expect("strategy log lock"),
[(
expected.kind().to_string(),
std::error::Error::source(&expected).map(ToString::to_string),
)]
);
}
#[test]
fn attempt_names() {
assert_eq!(attempt_name(Some("b"), RetryWrap::Child, 2), "attempt-2");
assert_eq!(
attempt_name(Some("b"), RetryWrap::Virtual, 2),
"b-attempt-2"
);
assert_eq!(attempt_name(Some(""), RetryWrap::Virtual, 2), "attempt-2");
assert_eq!(attempt_name(None, RetryWrap::Virtual, 2), "attempt-2");
}
}