use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;
use redis_tower_core::{Command, RedisError};
use tower_layer::Layer;
use tower_service::Service;
use crate::executor::{ExecutorService, RedisExecutor};
#[derive(Debug, Clone)]
pub struct RetryPolicy {
pub max_retries: usize,
pub base_delay: Duration,
pub max_delay: Duration,
pub jitter: bool,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_retries: 3,
base_delay: Duration::from_millis(50),
max_delay: Duration::from_secs(1),
jitter: true,
}
}
}
impl RetryPolicy {
#[must_use]
pub fn new(max_retries: usize) -> Self {
Self {
max_retries,
..Self::default()
}
}
#[must_use]
pub fn max_retries(mut self, n: usize) -> Self {
self.max_retries = n;
self
}
#[must_use]
pub fn base_delay(mut self, d: Duration) -> Self {
self.base_delay = d;
self
}
#[must_use]
pub fn max_delay(mut self, d: Duration) -> Self {
self.max_delay = d;
self
}
#[must_use]
pub fn jitter(mut self, enabled: bool) -> Self {
self.jitter = enabled;
self
}
pub fn should_retry(&self, idempotent: bool, err: &RedisError) -> bool {
idempotent && err.is_retryable()
}
fn delay_for_attempt(&self, attempt: usize) -> Duration {
let cap = self
.base_delay
.saturating_mul(1u32.wrapping_shl(attempt.min(31) as u32))
.min(self.max_delay);
if self.jitter {
let nanos = cap.as_nanos() as u64;
if nanos == 0 {
return Duration::ZERO;
}
Duration::from_nanos(rand::random::<u64>() % nanos)
} else {
cap
}
}
}
#[derive(Debug, Clone)]
pub struct RetryLayer {
policy: RetryPolicy,
}
impl RetryLayer {
pub fn new(policy: RetryPolicy) -> Self {
Self { policy }
}
}
impl<S> Layer<S> for RetryLayer {
type Service = RetryService<S>;
fn layer(&self, inner: S) -> Self::Service {
RetryService {
inner,
policy: self.policy.clone(),
}
}
}
#[derive(Debug, Clone)]
pub struct RetryService<S> {
inner: S,
policy: RetryPolicy,
}
impl<S> RetryService<S> {
pub fn new(inner: S, policy: RetryPolicy) -> Self {
Self { inner, policy }
}
}
impl<S, Cmd> Service<Cmd> for RetryService<S>
where
S: Service<Cmd, Response = Cmd::Response, Error = RedisError> + Clone + Send + 'static,
S::Future: Send,
Cmd: Command + Clone + Send + 'static,
{
type Response = Cmd::Response;
type Error = RedisError;
type Future = Pin<Box<dyn Future<Output = Result<Cmd::Response, RedisError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, cmd: Cmd) -> Self::Future {
let replacement = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, replacement);
let policy = self.policy.clone();
Box::pin(async move {
let idempotent = cmd.idempotent();
let deadline = cmd.deadline();
let mut attempt = 0usize;
let mut needs_readiness = false;
loop {
if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
return Err(RedisError::CommandTimeout);
}
if needs_readiness {
let ready = std::future::poll_fn(|cx| inner.poll_ready(cx));
match deadline {
Some(deadline) => tokio::time::timeout_at(deadline, ready)
.await
.map_err(|_elapsed| RedisError::CommandTimeout)??,
None => ready.await?,
}
}
let call = inner.call(cmd.clone());
let result = match deadline {
Some(deadline) => tokio::time::timeout_at(deadline, call)
.await
.map_err(|_elapsed| RedisError::CommandTimeout)?,
None => call.await,
};
match result {
Ok(response) => return Ok(response),
Err(err) => {
if attempt < policy.max_retries && policy.should_retry(idempotent, &err) {
let delay = policy.delay_for_attempt(attempt);
tracing::debug!(
attempt = attempt + 1,
max_retries = policy.max_retries,
delay = ?delay,
error = %err,
"redis: retrying idempotent command after retryable error"
);
let sleep = tokio::time::sleep(delay);
match deadline {
Some(deadline) => tokio::time::timeout_at(deadline, sleep)
.await
.map_err(|_elapsed| RedisError::CommandTimeout)?,
None => sleep.await,
}
attempt += 1;
needs_readiness = true;
continue;
}
return Err(err);
}
}
}
})
}
}
#[derive(Clone, Debug)]
pub struct RetryClient<C> {
inner: RetryService<ExecutorService<C>>,
}
impl<C> RetryClient<C>
where
C: RedisExecutor + Clone + Send + 'static,
{
pub fn new(client: C, policy: RetryPolicy) -> Self {
Self {
inner: RetryService::new(ExecutorService::new(client), policy),
}
}
pub async fn execute<Cmd>(&self, cmd: Cmd) -> Result<Cmd::Response, RedisError>
where
Cmd: Command + Clone + Send + 'static,
{
let deadline = cmd.deadline();
let mut svc = self.inner.clone();
let operation = async move {
std::future::poll_fn(|cx| {
<RetryService<ExecutorService<C>> as Service<Cmd>>::poll_ready(&mut svc, cx)
})
.await?;
<RetryService<ExecutorService<C>> as Service<Cmd>>::call(&mut svc, cmd).await
};
match deadline {
Some(deadline) => tokio::time::timeout_at(deadline, operation)
.await
.map_err(|_elapsed| RedisError::CommandTimeout)?,
None => operation.await,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use redis_tower_commands::{Get, Incr};
use redis_tower_core::WithDeadline;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Clone)]
struct FlakyExecutor {
calls: Arc<AtomicUsize>,
fail_times: usize,
error: Arc<dyn Fn() -> RedisError + Send + Sync>,
}
impl FlakyExecutor {
fn new(fail_times: usize, error: impl Fn() -> RedisError + Send + Sync + 'static) -> Self {
Self {
calls: Arc::new(AtomicUsize::new(0)),
fail_times,
error: Arc::new(error),
}
}
}
impl RedisExecutor for FlakyExecutor {
fn execute<Cmd: Command>(
&mut self,
cmd: Cmd,
) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
let n = self.calls.fetch_add(1, Ordering::SeqCst);
let fail = n < self.fail_times;
let err = (self.error)();
async move {
if fail {
Err(err)
} else {
cmd.parse_response(redis_tower_core::Frame::Null)
}
}
}
}
#[derive(Clone)]
struct SlowExecutor {
calls: Arc<AtomicUsize>,
delay: Duration,
}
impl RedisExecutor for SlowExecutor {
fn execute<Cmd: Command>(
&mut self,
cmd: Cmd,
) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
self.calls.fetch_add(1, Ordering::SeqCst);
let delay = self.delay;
async move {
tokio::time::sleep(delay).await;
cmd.parse_response(redis_tower_core::Frame::Null)
}
}
}
#[derive(Clone)]
struct ReadyForFirstAttemptService {
calls: Arc<AtomicUsize>,
}
impl Service<WithDeadline<Get>> for ReadyForFirstAttemptService {
type Response = Option<bytes::Bytes>;
type Error = RedisError;
type Future = std::future::Ready<Result<Self::Response, RedisError>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
if self.calls.load(Ordering::SeqCst) == 0 {
Poll::Ready(Ok(()))
} else {
Poll::Pending
}
}
fn call(&mut self, _cmd: WithDeadline<Get>) -> Self::Future {
self.calls.fetch_add(1, Ordering::SeqCst);
std::future::ready(Err(RedisError::ConnectionClosed))
}
}
fn no_jitter(max_retries: usize) -> RetryPolicy {
RetryPolicy::new(max_retries)
.base_delay(Duration::from_millis(0))
.jitter(false)
}
#[test]
fn default_policy_retries_only_idempotent_retryable() {
let policy = RetryPolicy::default();
assert!(policy.should_retry(true, &RedisError::ConnectionClosed));
assert!(!policy.should_retry(false, &RedisError::ConnectionClosed));
assert!(!policy.should_retry(true, &RedisError::Redis("WRONGTYPE".into())));
assert!(!policy.should_retry(false, &RedisError::Redis("WRONGTYPE".into())));
}
#[test]
fn delay_grows_exponentially_and_caps() {
let policy = RetryPolicy::new(5)
.base_delay(Duration::from_millis(10))
.max_delay(Duration::from_millis(35))
.jitter(false);
assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(10));
assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(20));
assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(35));
assert_eq!(policy.delay_for_attempt(9), Duration::from_millis(35));
}
#[tokio::test]
async fn retries_idempotent_command_until_success() {
let exec = FlakyExecutor::new(2, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let client = RetryClient::new(exec, no_jitter(3));
let result: Option<bytes::Bytes> = client.execute(Get::new("k")).await.unwrap();
assert_eq!(result, None);
assert_eq!(counter.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn gives_up_after_budget_exhausted() {
let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let client = RetryClient::new(exec, no_jitter(2));
let result = client.execute(Get::new("k")).await;
assert!(matches!(result, Err(RedisError::ConnectionClosed)));
assert_eq!(counter.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn does_not_retry_non_idempotent_command() {
let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let client = RetryClient::new(exec, no_jitter(5));
let result = client.execute(Incr::new("k")).await;
assert!(matches!(result, Err(RedisError::ConnectionClosed)));
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn does_not_retry_non_retryable_error() {
let exec = FlakyExecutor::new(usize::MAX, || RedisError::Redis("WRONGTYPE".into()));
let counter = exec.calls.clone();
let client = RetryClient::new(exec, no_jitter(5));
let result = client.execute(Get::new("k")).await;
assert!(matches!(result, Err(RedisError::Redis(_))));
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn zero_budget_disables_retries() {
let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let client = RetryClient::new(exec, no_jitter(0));
let result = client.execute(Get::new("k")).await;
assert!(result.is_err());
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn retry_layer_wraps_a_service() {
use tower_layer::Layer;
let exec = FlakyExecutor::new(1, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let mut svc = RetryLayer::new(no_jitter(3)).layer(ExecutorService::new(exec));
std::future::poll_fn(|cx| Service::<Get>::poll_ready(&mut svc, cx))
.await
.unwrap();
let result: Option<bytes::Bytes> = svc.call(Get::new("k")).await.unwrap();
assert_eq!(result, None);
assert_eq!(counter.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn absolute_deadline_bounds_retry_backoff() {
let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
let counter = exec.calls.clone();
let policy = RetryPolicy::new(5)
.base_delay(Duration::from_millis(200))
.max_delay(Duration::from_millis(200))
.jitter(false);
let client = RetryClient::new(exec, policy);
let result = client
.execute(WithDeadline::after(
Get::new("k"),
Duration::from_millis(20),
))
.await;
assert!(matches!(result, Err(RedisError::CommandTimeout)));
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn absolute_deadline_bounds_each_retry_attempt() {
let calls = Arc::new(AtomicUsize::new(0));
let client = RetryClient::new(
SlowExecutor {
calls: Arc::clone(&calls),
delay: Duration::from_millis(100),
},
no_jitter(3),
);
let result = client
.execute(WithDeadline::after(
Get::new("k"),
Duration::from_millis(20),
))
.await;
assert!(matches!(result, Err(RedisError::CommandTimeout)));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn absolute_deadline_bounds_inner_readiness() {
let calls = Arc::new(AtomicUsize::new(0));
let mut service = RetryService::new(
ReadyForFirstAttemptService {
calls: Arc::clone(&calls),
},
no_jitter(3),
);
std::future::poll_fn(|cx| Service::<WithDeadline<Get>>::poll_ready(&mut service, cx))
.await
.unwrap();
let result = service
.call(WithDeadline::after(
Get::new("k"),
Duration::from_millis(20),
))
.await;
assert!(matches!(result, Err(RedisError::CommandTimeout)));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[test]
fn retry_service_preserves_inner_backpressure() {
let calls = Arc::new(AtomicUsize::new(1));
let mut service = RetryService::new(
ReadyForFirstAttemptService {
calls: Arc::clone(&calls),
},
no_jitter(3),
);
let waker = futures::task::noop_waker();
let mut cx = Context::from_waker(&waker);
let readiness = Service::<WithDeadline<Get>>::poll_ready(&mut service, &mut cx);
assert!(readiness.is_pending());
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
}