redis-tower 0.1.2

A Tower-based Redis client with strong typing, composable middleware, and resilience
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
//! Idempotent-aware automatic retries at the command altitude.
//!
//! [`RetryLayer`] is a Tower [`Layer`] that wraps any
//! command-level `Service<Cmd>` (for example
//! [`ExecutorService`] over a real client, or a
//! [`CommandAdapter`](crate::CommandAdapter)) and retries failed commands
//! according to a [`RetryPolicy`].
//!
//! The retry decision runs at the **command altitude**, above the point where
//! a typed [`Command`] is lowered to a [`Frame`](redis_tower_core::Frame), so
//! the policy can read [`Command::idempotent`]. That matters: retrying a
//! non-idempotent write (INCR, LPUSH, ...) after a connection error can
//! silently duplicate data, because the first attempt may have reached Redis
//! before the connection dropped. The default policy therefore retries only
//! when the command is idempotent **and** the error is
//! [retryable](redis_tower_core::RedisError::is_retryable).
//!
//! This is a safety guarantee neither redis-rs nor fred offer: automatic
//! retries that are on by default only for the commands where they cannot
//! corrupt data.
//!
//! A command carrying a [`Command::deadline`] keeps that one absolute budget
//! across every retry. [`RetryService`] applies it while waiting for inner
//! readiness inside `call`, during each attempt, and during backoff, so retry
//! middleware cannot reset or sleep past the deadline regardless of layer
//! order. Tower's outer `poll_ready` runs before the service receives a
//! request, so request metadata cannot bound that caller-controlled wait;
//! [`RetryClient::execute`] owns the command early enough to bound both outer
//! readiness and the full retry call.
//!
//! # Two ways to use it
//!
//! Compose it as a Tower layer over a bridged client:
//!
//! ```no_run
//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
//! use redis_tower::{ExecutorService, MultiplexedClient, RetryLayer, RetryPolicy};
//! use tower::ServiceBuilder;
//!
//! let client = MultiplexedClient::connect("127.0.0.1:6379").await?;
//! let service = ServiceBuilder::new()
//!     .layer(RetryLayer::new(RetryPolicy::default()))
//!     .service(ExecutorService::new(client));
//! # let _ = service;
//! # Ok(())
//! # }
//! ```
//!
//! Or opt in directly on a client via its `retry` combinator, which returns a
//! [`RetryClient`] with an ergonomic `execute`:
//!
//! ```no_run
//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
//! use redis_tower::{MultiplexedClient, RetryPolicy};
//! use redis_tower::commands::Get;
//!
//! let client = MultiplexedClient::connect("127.0.0.1:6379").await?;
//! let retrying = client.retry(RetryPolicy::default());
//! let value: Option<bytes::Bytes> = retrying.execute(Get::new("key")).await?;
//! # let _ = value;
//! # Ok(())
//! # }
//! ```

use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::time::Duration;

use redis_tower_core::{Command, RedisError};
use tower_layer::Layer;
use tower_service::Service;

use crate::executor::{ExecutorService, RedisExecutor};

/// Policy controlling idempotent-aware retries.
///
/// A retry happens only when [`should_retry`](Self::should_retry) returns
/// `true` and the attempt budget ([`max_retries`](Self::max_retries)) is not
/// yet exhausted. The default decision is `idempotent && err.is_retryable()`:
/// safe automatic retries for read-only / naturally-idempotent commands, and
/// no silent write duplication for the rest.
///
/// Backoff mirrors the reconnect backoff: exponential
/// `base_delay * 2^attempt`, capped at `max_delay`, with optional full jitter.
#[derive(Debug, Clone)]
pub struct RetryPolicy {
    /// Maximum number of retries **after** the first attempt. A value of `0`
    /// disables retrying (one attempt only). Total attempts are
    /// `max_retries + 1`.
    pub max_retries: usize,
    /// Base delay used for the first retry; doubles each subsequent retry.
    pub base_delay: Duration,
    /// Upper bound on any single backoff delay.
    pub max_delay: Duration,
    /// Whether to apply full jitter to each backoff delay.
    ///
    /// When `true` (the default), each delay is a uniformly random value in
    /// `[0, cap)` where `cap` is the un-jittered exponential delay, spreading
    /// retries to avoid thundering-herd storms. When `false`, delays are the
    /// deterministic `base_delay * 2^attempt` (useful in tests).
    pub jitter: bool,
}

impl Default for RetryPolicy {
    fn default() -> Self {
        Self {
            max_retries: 3,
            base_delay: Duration::from_millis(50),
            max_delay: Duration::from_secs(1),
            jitter: true,
        }
    }
}

impl RetryPolicy {
    /// Create a policy with a given retry budget and otherwise-default backoff.
    #[must_use]
    pub fn new(max_retries: usize) -> Self {
        Self {
            max_retries,
            ..Self::default()
        }
    }

    /// Set the maximum number of retries after the first attempt.
    #[must_use]
    pub fn max_retries(mut self, n: usize) -> Self {
        self.max_retries = n;
        self
    }

    /// Set the base delay used for the first retry.
    #[must_use]
    pub fn base_delay(mut self, d: Duration) -> Self {
        self.base_delay = d;
        self
    }

    /// Set the cap applied to each backoff delay.
    #[must_use]
    pub fn max_delay(mut self, d: Duration) -> Self {
        self.max_delay = d;
        self
    }

    /// Enable or disable full jitter on backoff delays.
    #[must_use]
    pub fn jitter(mut self, enabled: bool) -> Self {
        self.jitter = enabled;
        self
    }

    /// Decide whether a failed command should be retried.
    ///
    /// The default policy retries only when the command is idempotent and the
    /// error is [retryable](RedisError::is_retryable). This is what makes
    /// automatic retries safe by default: a non-idempotent write is never
    /// re-sent after a connection error, so it cannot be silently duplicated.
    pub fn should_retry(&self, idempotent: bool, err: &RedisError) -> bool {
        idempotent && err.is_retryable()
    }

    /// Backoff delay before the retry with 0-based index `attempt` (the first
    /// retry is `attempt = 0`).
    fn delay_for_attempt(&self, attempt: usize) -> Duration {
        let cap = self
            .base_delay
            .saturating_mul(1u32.wrapping_shl(attempt.min(31) as u32))
            .min(self.max_delay);

        if self.jitter {
            let nanos = cap.as_nanos() as u64;
            if nanos == 0 {
                return Duration::ZERO;
            }
            Duration::from_nanos(rand::random::<u64>() % nanos)
        } else {
            cap
        }
    }
}

/// Tower [`Layer`] that wraps a command-level service with idempotent-aware
/// retries. See the [module docs](crate::retry).
#[derive(Debug, Clone)]
pub struct RetryLayer {
    policy: RetryPolicy,
}

impl RetryLayer {
    /// Create a retry layer from a [`RetryPolicy`].
    pub fn new(policy: RetryPolicy) -> Self {
        Self { policy }
    }
}

impl<S> Layer<S> for RetryLayer {
    type Service = RetryService<S>;

    fn layer(&self, inner: S) -> Self::Service {
        RetryService {
            inner,
            policy: self.policy.clone(),
        }
    }
}

/// A `Service<Cmd>` that retries failed commands per a [`RetryPolicy`].
///
/// Requires `Cmd: Clone` because each attempt re-issues a fresh copy of the
/// command. Produced by [`RetryLayer`], or constructed directly with
/// [`RetryService::new`]. The service continues to propagate its inner
/// readiness through [`Service::poll_ready`] for Tower backpressure. The
/// ready inner service is moved into [`Service::call`], and readiness is
/// reacquired before each retry. Those retry readiness waits are bounded by a
/// typed command's absolute deadline.
#[derive(Debug, Clone)]
pub struct RetryService<S> {
    inner: S,
    policy: RetryPolicy,
}

impl<S> RetryService<S> {
    /// Wrap an inner command-level service with retries.
    pub fn new(inner: S, policy: RetryPolicy) -> Self {
        Self { inner, policy }
    }
}

impl<S, Cmd> Service<Cmd> for RetryService<S>
where
    S: Service<Cmd, Response = Cmd::Response, Error = RedisError> + Clone + Send + 'static,
    S::Future: Send,
    Cmd: Command + Clone + Send + 'static,
{
    type Response = Cmd::Response;
    type Error = RedisError;
    type Future = Pin<Box<dyn Future<Output = Result<Cmd::Response, RedisError>> + Send>>;

    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        // Preserve the inner service's Tower backpressure. `call` moves this
        // exact ready instance into its future, and each retry reacquires
        // readiness there under the request deadline. The ergonomic
        // `RetryClient::execute` also bounds this outer readiness wait because
        // it already owns `cmd`.
        self.inner.poll_ready(cx)
    }

    fn call(&mut self, cmd: Cmd) -> Self::Future {
        // Move the exact service instance that `poll_ready` admitted into the
        // future, replacing it with a cheap clone for the next caller. This is
        // the standard Tower readiness-preserving clone pattern: cloning the
        // ready instance directly could discard a reserved capacity permit.
        let replacement = self.inner.clone();
        let mut inner = std::mem::replace(&mut self.inner, replacement);
        let policy = self.policy.clone();
        Box::pin(async move {
            let idempotent = cmd.idempotent();
            let deadline = cmd.deadline();
            let mut attempt = 0usize;
            let mut needs_readiness = false;
            loop {
                if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
                    return Err(RedisError::CommandTimeout);
                }

                if needs_readiness {
                    let ready = std::future::poll_fn(|cx| inner.poll_ready(cx));
                    match deadline {
                        Some(deadline) => tokio::time::timeout_at(deadline, ready)
                            .await
                            .map_err(|_elapsed| RedisError::CommandTimeout)??,
                        None => ready.await?,
                    }
                }

                let call = inner.call(cmd.clone());
                let result = match deadline {
                    Some(deadline) => tokio::time::timeout_at(deadline, call)
                        .await
                        .map_err(|_elapsed| RedisError::CommandTimeout)?,
                    None => call.await,
                };
                match result {
                    Ok(response) => return Ok(response),
                    Err(err) => {
                        if attempt < policy.max_retries && policy.should_retry(idempotent, &err) {
                            let delay = policy.delay_for_attempt(attempt);
                            tracing::debug!(
                                attempt = attempt + 1,
                                max_retries = policy.max_retries,
                                delay = ?delay,
                                error = %err,
                                "redis: retrying idempotent command after retryable error"
                            );
                            let sleep = tokio::time::sleep(delay);
                            match deadline {
                                Some(deadline) => tokio::time::timeout_at(deadline, sleep)
                                    .await
                                    .map_err(|_elapsed| RedisError::CommandTimeout)?,
                                None => sleep.await,
                            }
                            attempt += 1;
                            needs_readiness = true;
                            continue;
                        }
                        return Err(err);
                    }
                }
            }
        })
    }
}

/// An ergonomic retrying wrapper around a cloneable [`RedisExecutor`].
///
/// Returned by the `retry` combinator on [`MultiplexedClient`] and
/// [`ResilientRedisClient`]. It bridges the client to a command-level
/// [`Service`] via [`ExecutorService`] and applies a [`RetryService`], then
/// exposes a familiar `execute` that reissues idempotent commands on retryable
/// errors.
///
/// `execute` requires `Cmd: Clone` (every command builder derives it) because a
/// retry re-sends a copy of the command.
/// [`WithDeadline`](redis_tower_core::WithDeadline) bounds readiness, every
/// attempt, and backoff with one absolute instant; retrying never resets it.
///
/// [`MultiplexedClient`]: crate::MultiplexedClient
/// [`ResilientRedisClient`]: crate::ResilientRedisClient
#[derive(Clone, Debug)]
pub struct RetryClient<C> {
    inner: RetryService<ExecutorService<C>>,
}

impl<C> RetryClient<C>
where
    C: RedisExecutor + Clone + Send + 'static,
{
    /// Wrap a cloneable [`RedisExecutor`] with a [`RetryPolicy`].
    pub fn new(client: C, policy: RetryPolicy) -> Self {
        Self {
            inner: RetryService::new(ExecutorService::new(client), policy),
        }
    }

    /// Execute a command, retrying per the policy.
    ///
    /// Idempotent commands are re-issued on retryable errors up to the policy
    /// budget; non-idempotent commands are never retried, so a write cannot be
    /// silently duplicated.
    pub async fn execute<Cmd>(&self, cmd: Cmd) -> Result<Cmd::Response, RedisError>
    where
        Cmd: Command + Clone + Send + 'static,
    {
        let deadline = cmd.deadline();
        let mut svc = self.inner.clone();
        let operation = async move {
            std::future::poll_fn(|cx| {
                <RetryService<ExecutorService<C>> as Service<Cmd>>::poll_ready(&mut svc, cx)
            })
            .await?;
            <RetryService<ExecutorService<C>> as Service<Cmd>>::call(&mut svc, cmd).await
        };

        match deadline {
            Some(deadline) => tokio::time::timeout_at(deadline, operation)
                .await
                .map_err(|_elapsed| RedisError::CommandTimeout)?,
            None => operation.await,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use redis_tower_commands::{Get, Incr};
    use redis_tower_core::WithDeadline;
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};

    /// A `RedisExecutor` that fails the first `fail_times` calls with a
    /// configurable error, then succeeds, counting every call.
    #[derive(Clone)]
    struct FlakyExecutor {
        calls: Arc<AtomicUsize>,
        fail_times: usize,
        error: Arc<dyn Fn() -> RedisError + Send + Sync>,
    }

    impl FlakyExecutor {
        fn new(fail_times: usize, error: impl Fn() -> RedisError + Send + Sync + 'static) -> Self {
            Self {
                calls: Arc::new(AtomicUsize::new(0)),
                fail_times,
                error: Arc::new(error),
            }
        }
    }

    impl RedisExecutor for FlakyExecutor {
        fn execute<Cmd: Command>(
            &mut self,
            cmd: Cmd,
        ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
            let n = self.calls.fetch_add(1, Ordering::SeqCst);
            let fail = n < self.fail_times;
            let err = (self.error)();
            async move {
                if fail {
                    Err(err)
                } else {
                    // On success, parse a null frame -- fine for GET
                    // (`Option<Bytes>` -> None) and INCR is never reached
                    // here because it is non-idempotent and not retried.
                    cmd.parse_response(redis_tower_core::Frame::Null)
                }
            }
        }
    }

    #[derive(Clone)]
    struct SlowExecutor {
        calls: Arc<AtomicUsize>,
        delay: Duration,
    }

    impl RedisExecutor for SlowExecutor {
        fn execute<Cmd: Command>(
            &mut self,
            cmd: Cmd,
        ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
            self.calls.fetch_add(1, Ordering::SeqCst);
            let delay = self.delay;
            async move {
                tokio::time::sleep(delay).await;
                cmd.parse_response(redis_tower_core::Frame::Null)
            }
        }
    }

    #[derive(Clone)]
    struct ReadyForFirstAttemptService {
        calls: Arc<AtomicUsize>,
    }

    impl Service<WithDeadline<Get>> for ReadyForFirstAttemptService {
        type Response = Option<bytes::Bytes>;
        type Error = RedisError;
        type Future = std::future::Ready<Result<Self::Response, RedisError>>;

        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
            if self.calls.load(Ordering::SeqCst) == 0 {
                Poll::Ready(Ok(()))
            } else {
                Poll::Pending
            }
        }

        fn call(&mut self, _cmd: WithDeadline<Get>) -> Self::Future {
            self.calls.fetch_add(1, Ordering::SeqCst);
            std::future::ready(Err(RedisError::ConnectionClosed))
        }
    }

    fn no_jitter(max_retries: usize) -> RetryPolicy {
        RetryPolicy::new(max_retries)
            .base_delay(Duration::from_millis(0))
            .jitter(false)
    }

    #[test]
    fn default_policy_retries_only_idempotent_retryable() {
        let policy = RetryPolicy::default();
        // Idempotent + retryable -> retry.
        assert!(policy.should_retry(true, &RedisError::ConnectionClosed));
        // Non-idempotent + retryable -> do not retry (write-duplication guard).
        assert!(!policy.should_retry(false, &RedisError::ConnectionClosed));
        // Idempotent + non-retryable -> do not retry.
        assert!(!policy.should_retry(true, &RedisError::Redis("WRONGTYPE".into())));
        // Neither -> do not retry.
        assert!(!policy.should_retry(false, &RedisError::Redis("WRONGTYPE".into())));
    }

    #[test]
    fn delay_grows_exponentially_and_caps() {
        let policy = RetryPolicy::new(5)
            .base_delay(Duration::from_millis(10))
            .max_delay(Duration::from_millis(35))
            .jitter(false);
        assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(10));
        assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(20));
        // 40ms would exceed the 35ms cap.
        assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(35));
        assert_eq!(policy.delay_for_attempt(9), Duration::from_millis(35));
    }

    #[tokio::test]
    async fn retries_idempotent_command_until_success() {
        // Fail twice, then succeed: with a budget of 3 the third attempt wins.
        let exec = FlakyExecutor::new(2, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let client = RetryClient::new(exec, no_jitter(3));

        let result: Option<bytes::Bytes> = client.execute(Get::new("k")).await.unwrap();
        assert_eq!(result, None);
        // 1 initial attempt + 2 retries = 3 calls.
        assert_eq!(counter.load(Ordering::SeqCst), 3);
    }

    #[tokio::test]
    async fn gives_up_after_budget_exhausted() {
        // Always fails; budget of 2 retries means 3 attempts then an error.
        let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let client = RetryClient::new(exec, no_jitter(2));

        let result = client.execute(Get::new("k")).await;
        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
        assert_eq!(counter.load(Ordering::SeqCst), 3);
    }

    #[tokio::test]
    async fn does_not_retry_non_idempotent_command() {
        // INCR is non-idempotent: a single connection failure must not be
        // retried, or the counter could be double-incremented.
        let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let client = RetryClient::new(exec, no_jitter(5));

        let result = client.execute(Incr::new("k")).await;
        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
        // Exactly one attempt, no retries.
        assert_eq!(counter.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn does_not_retry_non_retryable_error() {
        // A WRONGTYPE is a caller error, not a transient fault: never retried
        // even for an idempotent command.
        let exec = FlakyExecutor::new(usize::MAX, || RedisError::Redis("WRONGTYPE".into()));
        let counter = exec.calls.clone();
        let client = RetryClient::new(exec, no_jitter(5));

        let result = client.execute(Get::new("k")).await;
        assert!(matches!(result, Err(RedisError::Redis(_))));
        assert_eq!(counter.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn zero_budget_disables_retries() {
        let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let client = RetryClient::new(exec, no_jitter(0));

        let result = client.execute(Get::new("k")).await;
        assert!(result.is_err());
        assert_eq!(counter.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn retry_layer_wraps_a_service() {
        // Exercise the Tower Layer path: RetryLayer over an ExecutorService.
        use tower_layer::Layer;

        let exec = FlakyExecutor::new(1, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let mut svc = RetryLayer::new(no_jitter(3)).layer(ExecutorService::new(exec));

        std::future::poll_fn(|cx| Service::<Get>::poll_ready(&mut svc, cx))
            .await
            .unwrap();
        let result: Option<bytes::Bytes> = svc.call(Get::new("k")).await.unwrap();
        assert_eq!(result, None);
        assert_eq!(counter.load(Ordering::SeqCst), 2);
    }

    #[tokio::test]
    async fn absolute_deadline_bounds_retry_backoff() {
        let exec = FlakyExecutor::new(usize::MAX, || RedisError::ConnectionClosed);
        let counter = exec.calls.clone();
        let policy = RetryPolicy::new(5)
            .base_delay(Duration::from_millis(200))
            .max_delay(Duration::from_millis(200))
            .jitter(false);
        let client = RetryClient::new(exec, policy);

        let result = client
            .execute(WithDeadline::after(
                Get::new("k"),
                Duration::from_millis(20),
            ))
            .await;

        assert!(matches!(result, Err(RedisError::CommandTimeout)));
        assert_eq!(counter.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn absolute_deadline_bounds_each_retry_attempt() {
        let calls = Arc::new(AtomicUsize::new(0));
        let client = RetryClient::new(
            SlowExecutor {
                calls: Arc::clone(&calls),
                delay: Duration::from_millis(100),
            },
            no_jitter(3),
        );

        let result = client
            .execute(WithDeadline::after(
                Get::new("k"),
                Duration::from_millis(20),
            ))
            .await;

        assert!(matches!(result, Err(RedisError::CommandTimeout)));
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn absolute_deadline_bounds_inner_readiness() {
        let calls = Arc::new(AtomicUsize::new(0));
        let mut service = RetryService::new(
            ReadyForFirstAttemptService {
                calls: Arc::clone(&calls),
            },
            no_jitter(3),
        );

        std::future::poll_fn(|cx| Service::<WithDeadline<Get>>::poll_ready(&mut service, cx))
            .await
            .unwrap();
        let result = service
            .call(WithDeadline::after(
                Get::new("k"),
                Duration::from_millis(20),
            ))
            .await;

        assert!(matches!(result, Err(RedisError::CommandTimeout)));
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }

    #[test]
    fn retry_service_preserves_inner_backpressure() {
        let calls = Arc::new(AtomicUsize::new(1));
        let mut service = RetryService::new(
            ReadyForFirstAttemptService {
                calls: Arc::clone(&calls),
            },
            no_jitter(3),
        );
        let waker = futures::task::noop_waker();
        let mut cx = Context::from_waker(&waker);

        let readiness = Service::<WithDeadline<Get>>::poll_ready(&mut service, &mut cx);

        assert!(readiness.is_pending());
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }
}