Skip to main content

redis_tower/
resilient.rs

1//! Batteries-included resilient Redis client.
2//!
3//! [`ResilientRedisClient`] combines shared access (`Arc<Mutex<>>`) with
4//! automatic reconnection on connection loss. It is the recommended
5//! client for long-running applications that need to survive transient
6//! network failures without manual intervention.
7
8use std::collections::VecDeque;
9use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
10use std::sync::{Arc, Mutex as StdMutex};
11use std::time::{Duration, Instant};
12
13use redis_tower_commands::Ping;
14use redis_tower_core::{Command, ConnectionConfig, Frame, RedisConnection, RedisError};
15use tokio::sync::{Mutex, Notify, watch};
16use tower_service::Service;
17
18use crate::circuit_breaker::{RedisCircuitBreakerClient, RedisCircuitBreakerConfig};
19use crate::reconnect::{
20    AddrConnectionFactory, ConnectionDisconnectReason, ConnectionEvent, ConnectionEventBus,
21    ConnectionFactory, ReconnectConfig, UrlConnectionFactory, publish_disconnect_once,
22};
23use crate::retry::{RetryClient, RetryPolicy};
24
25/// Configuration for the opt-in queue used while Redis is reconnecting.
26///
27/// The queue is disabled unless this config is passed to
28/// [`ResilientRedisClient::connect_with_offline_queue`] or
29/// [`ResilientRedisClient::with_config_and_offline_queue`]. It admits only
30/// commands whose [`Command::idempotent`] implementation returns `true`.
31/// Queue admission is non-blocking: once `capacity` requests are admitted,
32/// another idempotent request fails immediately with [`RedisError::QueueFull`].
33/// Admission is serialized under the queue lock, so commands replay in ticket
34/// order. Futures first polled concurrently can be admitted in either scheduler
35/// order; the queue does not claim an ordering before admission.
36///
37/// Each command gets at most three wire replays by default. This prevents a
38/// server that repeatedly accepts a connection and immediately drops the
39/// replay from keeping the head ticket alive forever. Use
40/// [`with_max_replay_attempts`](Self::with_max_replay_attempts) to tune that
41/// separate replay budget. Reconnect attempts themselves remain governed by
42/// [`ReconnectConfig`].
43///
44/// A capacity of zero is valid and provides fail-fast reconnect behavior while
45/// still running reconnect campaigns in the background.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub struct OfflineQueueConfig {
48    capacity: usize,
49    max_replay_attempts: usize,
50}
51
52impl OfflineQueueConfig {
53    /// Create an offline queue that holds at most `capacity` commands.
54    #[must_use]
55    pub const fn new(capacity: usize) -> Self {
56        Self {
57            capacity,
58            max_replay_attempts: 3,
59        }
60    }
61
62    /// Return the maximum number of commands that may wait for reconnection.
63    #[must_use]
64    pub const fn capacity(&self) -> usize {
65        self.capacity
66    }
67
68    /// Set the maximum number of replacement-connection wire attempts for one
69    /// admitted command.
70    ///
71    /// This must be non-zero. Once the limit is reached, the command returns
72    /// [`RedisError::ReconnectFailed`] with the final wire error. The tainted
73    /// connection is still replaced before a later queued command can run.
74    #[must_use]
75    pub const fn with_max_replay_attempts(mut self, max_replay_attempts: usize) -> Self {
76        assert!(
77            max_replay_attempts > 0,
78            "offline queue replay attempts must be non-zero"
79        );
80        self.max_replay_attempts = max_replay_attempts;
81        self
82    }
83
84    /// Return the per-command replacement-connection replay limit.
85    #[must_use]
86    pub const fn max_replay_attempts(&self) -> usize {
87        self.max_replay_attempts
88    }
89}
90
91impl Default for OfflineQueueConfig {
92    fn default() -> Self {
93        Self::new(1024)
94    }
95}
96
97/// A shared, auto-reconnecting Redis client.
98///
99/// Wraps a [`RedisConnection`] with automatic reconnection on connection
100/// loss. Uses `Arc<Mutex<>>` for cross-task sharing.
101///
102/// # Concurrency
103///
104/// `ResilientRedisClient` is `Clone + Send + Sync`. All clones share the same
105/// `Arc<Mutex<RedisConnection>>`, serializing commands one at a time.
106/// Reconnection is triggered only when a command fails with a connection error
107/// (`is_connection_error()` returns true); non-connection errors (WRONGTYPE,
108/// etc.) are returned to the caller without triggering reconnection. After
109/// the first reconnect attempt plus `max_retries` additional attempts are
110/// exhausted, the error propagates to the caller; the client is not permanently
111/// broken and will attempt reconnection on the next command.
112///
113/// # Queueing During Reconnection
114///
115/// The default remains fail-fast: the command that discovers a lost connection
116/// receives that error, and commands that reach the broken connection while a
117/// reconnect is running also fail. Use
118/// [`connect_with_offline_queue`](Self::connect_with_offline_queue) or
119/// [`with_config_and_offline_queue`](Self::with_config_and_offline_queue) to
120/// opt into a bounded shared queue. In that mode:
121///
122/// - only commands marked [`Command::idempotent`] wait and replay;
123/// - accepted commands replay in queue-ticket order after one shared reconnect
124///   campaign (concurrently first-polled futures can be admitted in either
125///   scheduler order);
126/// - overflow fails immediately with [`RedisError::QueueFull`]; and
127/// - non-idempotent commands fail with a connection error instead of risking a
128///   duplicate side effect.
129///
130/// Dropping all client clones cancels an in-progress reconnect, including a
131/// factory future with no configured connect timeout. Dropping a queued command
132/// future removes its ticket without blocking the commands behind it. Once a
133/// command has reached the wire, cancellation conservatively quarantines that
134/// socket and starts a replacement campaign before the next ticket can run, so
135/// a late response cannot be mistaken for the next command's response.
136///
137/// Tower users who work directly with [`Frame`] can instead put a bounded
138/// `tower::buffer::Buffer` in front of [`ReconnectService`](crate::ReconnectService):
139///
140/// ```no_run
141/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
142/// use redis_tower::ReconnectService;
143/// use redis_tower::reconnect::{AddrConnectionFactory, ReconnectConfig};
144/// use tower::buffer::Buffer;
145///
146/// let reconnecting = ReconnectService::new(
147///     AddrConnectionFactory::new("127.0.0.1:6379"),
148///     ReconnectConfig::default(),
149/// ).await?;
150/// let buffered = Buffer::new(reconnecting, 64);
151/// # let _ = buffered;
152/// # Ok(())
153/// # }
154/// ```
155///
156/// A frame buffer cannot inspect [`Command::idempotent`], so callers remain
157/// responsible for deciding whether retrying a failed frame is safe.
158///
159/// # Example
160///
161/// ```no_run
162/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
163/// use redis_tower::ResilientRedisClient;
164/// use redis_tower::commands::Set;
165///
166/// let client = ResilientRedisClient::connect("127.0.0.1:6379").await?;
167///
168/// let c = client.clone();
169/// tokio::spawn(async move {
170///     c.execute(Set::new("key", "value")).await.unwrap();
171/// });
172/// # Ok(())
173/// # }
174/// ```
175#[derive(Clone)]
176pub struct ResilientRedisClient {
177    conn: Arc<Mutex<RedisConnection>>,
178    factory: Arc<dyn ConnectionFactory>,
179    config: ReconnectConfig,
180    /// Single-flights reconnects across clones: a connection drop seen by many
181    /// clones triggers one reconnect, not one storm per clone.
182    gate: Arc<ReconnectGate>,
183    /// Present only for clients constructed with explicit offline queueing.
184    offline_queue: Option<OfflineQueueHandle>,
185    connection_events: Option<ConnectionEventBus>,
186    disconnect_reported: Option<Arc<AtomicBool>>,
187    /// Counts only client handles. Detached reconnect tasks clone the control,
188    /// not this lease, so final-handle shutdown remains observable immediately.
189    _lifecycle: Option<ResilientClientLifecycle>,
190}
191
192impl ResilientRedisClient {
193    /// Connect to Redis with default reconnection settings.
194    pub async fn connect(addr: &str) -> Result<Self, RedisError> {
195        Self::with_config(AddrConnectionFactory::new(addr), ReconnectConfig::default()).await
196    }
197
198    /// Connect to Redis and publish connection lifecycle events.
199    ///
200    /// Subscribe to `events` before calling this constructor to observe the
201    /// initial [`ConnectionEvent::Connected`] or
202    /// [`ConnectionEvent::ConnectFailed`] event.
203    pub async fn connect_with_events(
204        addr: &str,
205        events: ConnectionEventBus,
206    ) -> Result<Self, RedisError> {
207        Self::with_config_and_events(
208            AddrConnectionFactory::new(addr),
209            ReconnectConfig::default(),
210            events,
211        )
212        .await
213    }
214
215    /// Connect to Redis and enable a bounded queue while reconnecting.
216    ///
217    /// The queue is shared by every clone. Only idempotent commands are
218    /// admitted; see [`OfflineQueueConfig`] for overflow behavior.
219    pub async fn connect_with_offline_queue(
220        addr: &str,
221        queue_config: OfflineQueueConfig,
222    ) -> Result<Self, RedisError> {
223        Self::with_config_and_offline_queue(
224            AddrConnectionFactory::new(addr),
225            ReconnectConfig::default(),
226            queue_config,
227        )
228        .await
229    }
230
231    /// Connect to Redis with both an offline queue and lifecycle events.
232    pub async fn connect_with_offline_queue_and_events(
233        addr: &str,
234        queue_config: OfflineQueueConfig,
235        events: ConnectionEventBus,
236    ) -> Result<Self, RedisError> {
237        Self::with_config_and_offline_queue_and_events(
238            AddrConnectionFactory::new(addr),
239            ReconnectConfig::default(),
240            queue_config,
241            events,
242        )
243        .await
244    }
245
246    /// Connect with settings that are retained across every reconnect.
247    pub async fn connect_with_connection_config(
248        addr: &str,
249        connection_config: ConnectionConfig,
250    ) -> Result<Self, RedisError> {
251        Self::with_config(
252            AddrConnectionFactory::new(addr).with_connection_config(connection_config),
253            ReconnectConfig::default(),
254        )
255        .await
256    }
257
258    /// Connect via a Redis URL with default reconnection settings.
259    pub async fn connect_url(url: &str) -> Result<Self, RedisError> {
260        Self::with_config(UrlConnectionFactory::new(url), ReconnectConfig::default()).await
261    }
262
263    /// Connect via a Redis URL and enable a bounded queue while reconnecting.
264    pub async fn connect_url_with_offline_queue(
265        url: &str,
266        queue_config: OfflineQueueConfig,
267    ) -> Result<Self, RedisError> {
268        Self::with_config_and_offline_queue(
269            UrlConnectionFactory::new(url),
270            ReconnectConfig::default(),
271            queue_config,
272        )
273        .await
274    }
275
276    /// Connect from a Redis URL with settings retained across every reconnect.
277    pub async fn connect_url_with_connection_config(
278        url: &str,
279        connection_config: ConnectionConfig,
280    ) -> Result<Self, RedisError> {
281        Self::with_config(
282            UrlConnectionFactory::new(url).with_connection_config(connection_config),
283            ReconnectConfig::default(),
284        )
285        .await
286    }
287
288    /// Connect with a custom factory and reconnection config.
289    pub async fn with_config(
290        factory: impl ConnectionFactory,
291        config: ReconnectConfig,
292    ) -> Result<Self, RedisError> {
293        Self::build(factory, config, None, None).await
294    }
295
296    /// Connect with a custom factory and publish connection lifecycle events.
297    pub async fn with_config_and_events(
298        factory: impl ConnectionFactory,
299        config: ReconnectConfig,
300        events: ConnectionEventBus,
301    ) -> Result<Self, RedisError> {
302        Self::build(factory, config, None, Some(events)).await
303    }
304
305    /// Connect with a custom factory, reconnection policy, and offline queue.
306    ///
307    /// Unlike [`Self::with_config`], this enables idempotent command replay
308    /// after a successful reconnect. The initial connection is still
309    /// established before this function returns.
310    pub async fn with_config_and_offline_queue(
311        factory: impl ConnectionFactory,
312        config: ReconnectConfig,
313        queue_config: OfflineQueueConfig,
314    ) -> Result<Self, RedisError> {
315        Self::build(factory, config, Some(queue_config), None).await
316    }
317
318    /// Connect with custom reconnect, offline queue, and lifecycle settings.
319    pub async fn with_config_and_offline_queue_and_events(
320        factory: impl ConnectionFactory,
321        config: ReconnectConfig,
322        queue_config: OfflineQueueConfig,
323        events: ConnectionEventBus,
324    ) -> Result<Self, RedisError> {
325        Self::build(factory, config, Some(queue_config), Some(events)).await
326    }
327
328    async fn build(
329        factory: impl ConnectionFactory,
330        config: ReconnectConfig,
331        queue_config: Option<OfflineQueueConfig>,
332        connection_events: Option<ConnectionEventBus>,
333    ) -> Result<Self, RedisError> {
334        let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
335        let conn = match connect_with_timeout(&*factory, config.connect_timeout).await {
336            Ok(conn) => conn,
337            Err(error) => {
338                if let Some(events) = &connection_events {
339                    events.publish_with(|| ConnectionEvent::ConnectFailed {
340                        error: Arc::from(error.to_string()),
341                    });
342                }
343                return Err(error);
344            }
345        };
346        if let Some(events) = &connection_events {
347            events.publish(ConnectionEvent::Connected);
348        }
349        let disconnect_reported = connection_events
350            .as_ref()
351            .map(|_| Arc::new(AtomicBool::new(false)));
352        let lifecycle = connection_events
353            .as_ref()
354            .map(|events| ResilientClientLifecycle::new(events.clone()));
355        Ok(Self {
356            conn: Arc::new(Mutex::new(conn)),
357            factory,
358            config,
359            gate: Arc::new(ReconnectGate::new()),
360            offline_queue: queue_config.map(OfflineQueueHandle::new),
361            connection_events,
362            disconnect_reported,
363            _lifecycle: lifecycle,
364        })
365    }
366
367    /// Execute a command, reconnecting if the connection is lost.
368    ///
369    /// # Retry Safety
370    ///
371    /// Without an offline queue, a connection error starts reconnection but is
372    /// returned to the caller. If the caller retries, the original command may
373    /// already have executed before the socket dropped.
374    ///
375    /// With an offline queue, commands marked [`Command::idempotent`] wait for
376    /// reconnection and replay automatically. A non-idempotent command still
377    /// receives the original connection error and is never replayed. A
378    /// deadline carried by [`redis_tower_core::WithDeadline`] bounds queue
379    /// waiting, connection acquisition, and every wire attempt under one
380    /// absolute budget. Expiry removes the queue ticket; expiry after dispatch
381    /// also quarantines the socket before another command can use it.
382    pub async fn execute<Cmd: Command>(&self, cmd: Cmd) -> Result<Cmd::Response, RedisError> {
383        if let Some(queue) = &self.offline_queue {
384            let deadline = cmd.deadline();
385            if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
386                return Err(RedisError::CommandTimeout);
387            }
388            let operation = self.execute_with_offline_queue(cmd, queue);
389            return match deadline {
390                Some(deadline) => tokio::time::timeout_at(deadline, operation)
391                    .await
392                    .map_err(|_elapsed| RedisError::CommandTimeout)?,
393                None => operation.await,
394            };
395        }
396
397        let mut conn = self.conn.lock().await;
398        let result = conn.execute(cmd).await;
399
400        if let Err(ref e) = result
401            && e.is_connection_error()
402        {
403            self.publish_disconnect(e);
404            drop(conn);
405            self.reconnect().await;
406        }
407
408        result
409    }
410
411    /// Return the number of commands currently admitted to the offline queue.
412    ///
413    /// This is a point-in-time observability value. It returns zero when
414    /// offline queueing was not enabled.
415    #[must_use]
416    pub fn offline_queue_depth(&self) -> usize {
417        self.offline_queue
418            .as_ref()
419            .map_or(0, OfflineQueueHandle::depth)
420    }
421
422    /// Return whether this client currently has a reconnect campaign running.
423    #[must_use]
424    pub fn is_reconnecting(&self) -> bool {
425        self.offline_queue
426            .as_ref()
427            .is_some_and(OfflineQueueHandle::is_reconnecting)
428            || self.gate.is_reconnecting()
429    }
430
431    async fn execute_with_offline_queue<Cmd: Command>(
432        &self,
433        mut cmd: Cmd,
434        queue: &OfflineQueueHandle,
435    ) -> Result<Cmd::Response, RedisError> {
436        let idempotent = cmd.idempotent();
437        // Freeze the exact wire request before its first send. An idempotent
438        // replay must not rerun a command builder that uses interior state.
439        let frame = cmd.to_frame();
440
441        match queue.before_execute(idempotent) {
442            QueueDecision::Execute => {}
443            QueueDecision::Wait { permit, campaign } => {
444                if let Some(campaign) = campaign {
445                    self.spawn_reconnect_campaign(queue, campaign);
446                }
447                return self.execute_queued(cmd, frame, queue, permit).await;
448            }
449            QueueDecision::Reject { error, campaign } => {
450                if let Some(campaign) = campaign {
451                    self.spawn_reconnect_campaign(queue, campaign);
452                }
453                return Err(error.into_redis_error());
454            }
455        }
456
457        let mut conn = self.conn.lock().await;
458
459        // The phase can change while this caller waits for the connection
460        // mutex. Recheck before touching a connection another caller has
461        // already declared dead.
462        match queue.before_execute(idempotent) {
463            QueueDecision::Execute => {}
464            QueueDecision::Wait { permit, campaign } => {
465                drop(conn);
466                if let Some(campaign) = campaign {
467                    self.spawn_reconnect_campaign(queue, campaign);
468                }
469                return self.execute_queued(cmd, frame, queue, permit).await;
470            }
471            QueueDecision::Reject { error, campaign } => {
472                drop(conn);
473                if let Some(campaign) = campaign {
474                    self.spawn_reconnect_campaign(queue, campaign);
475                }
476                return Err(error.into_redis_error());
477            }
478        }
479
480        let mut alignment = ProtocolAlignmentGuard::new(self, queue);
481        let (returned_cmd, result) = execute_retained(&mut conn, cmd, frame.clone()).await;
482        alignment.disarm();
483        cmd = returned_cmd;
484        let Err(error) = result else {
485            return result;
486        };
487        if !error.is_connection_error() {
488            return Err(error);
489        }
490
491        self.publish_disconnect(&error);
492        let decision = queue.after_connection_error(idempotent);
493        drop(conn);
494        match decision {
495            QueueDecision::Wait { permit, campaign } => {
496                if let Some(campaign) = campaign {
497                    self.spawn_reconnect_campaign(queue, campaign);
498                }
499                self.execute_queued(cmd, frame, queue, permit).await
500            }
501            QueueDecision::Reject {
502                error: rejection,
503                campaign,
504            } => {
505                if let Some(campaign) = campaign {
506                    self.spawn_reconnect_campaign(queue, campaign);
507                }
508                // Preserve the error from the command that actually
509                // discovered the disconnect. QueueFull is the one useful
510                // exception: capacity zero must remain observable.
511                if matches!(rejection, QueueRejection::Full) {
512                    Err(RedisError::QueueFull)
513                } else {
514                    Err(error)
515                }
516            }
517            QueueDecision::Execute => unreachable!("a disconnect must leave offline mode"),
518        }
519    }
520
521    async fn execute_queued<Cmd: Command>(
522        &self,
523        mut cmd: Cmd,
524        frame: Frame,
525        queue: &OfflineQueueHandle,
526        permit: QueuePermit,
527    ) -> Result<Cmd::Response, RedisError> {
528        let mut replay_attempts = 0;
529        loop {
530            permit.wait_for_turn().await?;
531            let mut conn = self.conn.lock().await;
532
533            // A raw Service call or the preceding queued request can discover
534            // another disconnect between our wake-up and acquiring the
535            // connection. Never send until this ticket is still the active
536            // head of a draining queue.
537            match permit.check_turn() {
538                TurnState::Ready => {}
539                TurnState::Wait => {
540                    drop(conn);
541                    continue;
542                }
543                TurnState::Rejected(error) => return Err(error.into_redis_error()),
544            }
545
546            let mut alignment = ProtocolAlignmentGuard::new(self, queue);
547            let (returned_cmd, result) = execute_retained(&mut conn, cmd, frame.clone()).await;
548            alignment.disarm();
549            cmd = returned_cmd;
550            let Err(error) = result else {
551                return result;
552            };
553            if !error.is_connection_error() {
554                return Err(error);
555            }
556
557            self.publish_disconnect(&error);
558            let campaign = queue.reconnect_existing(&permit);
559            replay_attempts += 1;
560            drop(conn);
561            if let Some(campaign) = campaign {
562                self.spawn_reconnect_campaign(queue, campaign);
563            }
564            if replay_attempts >= queue.max_replay_attempts() {
565                return Err(RedisError::ReconnectFailed {
566                    attempts: replay_attempts,
567                    last_error: Arc::new(error),
568                });
569            }
570            // Keep the same ticket at the head. The command is idempotent, so
571            // it can safely cross another reconnect if the replacement socket
572            // dies before or during replay, up to the configured replay limit.
573        }
574    }
575
576    fn spawn_reconnect_campaign(&self, queue: &OfflineQueueHandle, campaign: u64) {
577        let factory = Arc::clone(&self.factory);
578        let config = self.config.clone();
579        let conn = Arc::downgrade(&self.conn);
580        let queue = Arc::downgrade(&queue.inner);
581        let lifecycle = self
582            ._lifecycle
583            .as_ref()
584            .map(ResilientClientLifecycle::control);
585        let disconnect_reported = self.disconnect_reported.clone();
586        let mut closed = queue
587            .upgrade()
588            .expect("offline queue owner exists while starting reconnect")
589            .closed_tx
590            .subscribe();
591
592        tokio::spawn(async move {
593            let outcome = reconnect_campaign_until_closed(
594                &*factory,
595                &config,
596                &mut closed,
597                lifecycle.as_deref(),
598            )
599            .await;
600            let Some(queue) = queue.upgrade() else {
601                return;
602            };
603
604            match outcome {
605                CancellableReconnect::Connected {
606                    connection: new_conn,
607                    attempts,
608                    elapsed,
609                } => {
610                    let Some(conn) = conn.upgrade() else {
611                        return;
612                    };
613                    let lock = conn.lock();
614                    tokio::pin!(lock);
615                    let mut conn = tokio::select! {
616                        result = closed.changed() => {
617                            let _ = result;
618                            return;
619                        }
620                        conn = &mut lock => conn,
621                    };
622                    if queue.campaign_is_active(campaign) {
623                        *conn = new_conn;
624                        if let Some(reported) = &disconnect_reported {
625                            reported.store(false, Ordering::Release);
626                        }
627                        if let Some(lifecycle) = &lifecycle {
628                            lifecycle.publish(ConnectionEvent::Reconnected { attempts, elapsed });
629                        }
630                        queue.finish_reconnect(campaign, None);
631                    }
632                }
633                CancellableReconnect::Failed {
634                    attempts,
635                    last_error,
636                } => {
637                    queue.finish_reconnect(
638                        campaign,
639                        Some(ReconnectFailure {
640                            attempts,
641                            last_error: Arc::new(last_error),
642                        }),
643                    );
644                }
645                CancellableReconnect::Closed => {}
646            }
647        });
648    }
649
650    fn publish_disconnect(&self, error: &RedisError) {
651        publish_disconnect_once(
652            self.connection_events.as_ref(),
653            self.disconnect_reported.as_deref(),
654            error,
655        );
656    }
657
658    /// Send a PING to verify the connection is alive.
659    ///
660    /// Returns `Ok(())` on success. Useful for Kubernetes readiness probes
661    /// and `/health` endpoints.
662    pub async fn health_check(&self) -> Result<(), RedisError> {
663        self.execute(Ping::new()).await?;
664        Ok(())
665    }
666
667    /// Wrap this client in idempotent-aware automatic retries.
668    ///
669    /// Returns a [`RetryClient`](crate::RetryClient) whose `execute` reissues
670    /// idempotent commands on retryable errors per the
671    /// [`RetryPolicy`](crate::RetryPolicy). Reconnection already happens
672    /// underneath on connection loss; layering retries on top means an
673    /// idempotent command that failed with a connection error is
674    /// automatically re-sent after the client reconnects, up to the policy
675    /// budget. Non-idempotent writes are never retried.
676    ///
677    /// # Example
678    ///
679    /// ```no_run
680    /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
681    /// use redis_tower::{ResilientRedisClient, RetryPolicy};
682    /// use redis_tower::commands::Get;
683    ///
684    /// let client = ResilientRedisClient::connect("127.0.0.1:6379").await?;
685    /// let retrying = client.retry(RetryPolicy::default());
686    /// let value: Option<bytes::Bytes> = retrying.execute(Get::new("key")).await?;
687    /// # let _ = value;
688    /// # Ok(())
689    /// # }
690    /// ```
691    pub fn retry(&self, policy: RetryPolicy) -> RetryClient<Self> {
692        RetryClient::new(self.clone(), policy)
693    }
694
695    /// Protect this reconnecting client with a Redis-aware circuit breaker.
696    ///
697    /// The breaker sits outside reconnection: connection and timeout failures
698    /// affect circuit health, while Redis command errors such as `WRONGTYPE`
699    /// do not. The returned client retains typed `execute`, `health_check`, and
700    /// idempotent-aware `retry` helpers.
701    pub fn with_circuit_breaker(
702        self,
703        config: RedisCircuitBreakerConfig,
704    ) -> RedisCircuitBreakerClient<Self> {
705        RedisCircuitBreakerClient::new(self, config)
706    }
707
708    /// Attempt to reconnect, single-flighting across clones.
709    async fn reconnect(&self) {
710        // Snapshot the generation before taking the gate. If another clone
711        // reconnects while we wait, the generation advances and we skip --
712        // a shared connection drop triggers one reconnect, not one per clone.
713        let seen = self.gate.generation();
714        let _guard = self.gate.enter().await;
715        if self.gate.generation() != seen {
716            return;
717        }
718
719        let _active = self.gate.mark_reconnecting();
720        let lifecycle = self._lifecycle.as_ref().map(|lease| lease.control.as_ref());
721        if let Some(success) = reconnect_campaign(&*self.factory, &self.config, lifecycle).await {
722            *self.conn.lock().await = success.connection;
723            if let Some(reported) = &self.disconnect_reported {
724                reported.store(false, Ordering::Release);
725            }
726            if let Some(lifecycle) = lifecycle {
727                lifecycle.publish(ConnectionEvent::Reconnected {
728                    attempts: success.attempts,
729                    elapsed: success.elapsed,
730                });
731            }
732            self.gate.mark_reconnected();
733        }
734    }
735}
736
737struct ResilientLifecycleControl {
738    state: StdMutex<ResilientLifecycleState>,
739    events: ConnectionEventBus,
740}
741
742struct ResilientLifecycleState {
743    handles: usize,
744    shutdown_published: bool,
745}
746
747impl ResilientLifecycleControl {
748    fn new(events: ConnectionEventBus) -> Self {
749        Self {
750            state: StdMutex::new(ResilientLifecycleState {
751                handles: 1,
752                shutdown_published: false,
753            }),
754            events,
755        }
756    }
757
758    fn state(&self) -> std::sync::MutexGuard<'_, ResilientLifecycleState> {
759        self.state
760            .lock()
761            .unwrap_or_else(std::sync::PoisonError::into_inner)
762    }
763
764    fn add_handle(&self) {
765        let mut state = self.state();
766        state.handles = state
767            .handles
768            .checked_add(1)
769            .expect("ResilientRedisClient handle count overflowed");
770    }
771
772    fn release_handle(&self) {
773        let mut state = self.state();
774        state.handles = state
775            .handles
776            .checked_sub(1)
777            .expect("ResilientRedisClient handle count underflowed");
778        if state.handles == 0 && !state.shutdown_published {
779            state.shutdown_published = true;
780            self.events.publish(ConnectionEvent::Disconnected {
781                reason: ConnectionDisconnectReason::Shutdown,
782            });
783        }
784    }
785
786    /// Publish while holding the same lock used by final-handle release. An
787    /// event that wins this lock is ordered before shutdown; one that loses to
788    /// final release is suppressed, making `Shutdown` terminal.
789    fn publish_with(&self, make_event: impl FnOnce() -> ConnectionEvent) -> bool {
790        let state = self.state();
791        if state.handles == 0 {
792            return false;
793        }
794        self.events.publish_with(make_event);
795        true
796    }
797
798    fn publish(&self, event: ConnectionEvent) -> bool {
799        self.publish_with(|| event)
800    }
801}
802
803struct ResilientClientLifecycle {
804    control: Arc<ResilientLifecycleControl>,
805}
806
807impl ResilientClientLifecycle {
808    fn new(events: ConnectionEventBus) -> Self {
809        Self {
810            control: Arc::new(ResilientLifecycleControl::new(events)),
811        }
812    }
813
814    fn control(&self) -> Arc<ResilientLifecycleControl> {
815        Arc::clone(&self.control)
816    }
817}
818
819impl Clone for ResilientClientLifecycle {
820    fn clone(&self) -> Self {
821        self.control.add_handle();
822        Self {
823            control: Arc::clone(&self.control),
824        }
825    }
826}
827
828impl Drop for ResilientClientLifecycle {
829    fn drop(&mut self) {
830        self.control.release_handle();
831    }
832}
833
834impl Service<Frame> for ResilientRedisClient {
835    type Response = Frame;
836    type Error = RedisError;
837    type Future =
838        std::pin::Pin<Box<dyn std::future::Future<Output = Result<Frame, RedisError>> + Send>>;
839
840    fn poll_ready(
841        &mut self,
842        _cx: &mut std::task::Context<'_>,
843    ) -> std::task::Poll<Result<(), Self::Error>> {
844        std::task::Poll::Ready(Ok(()))
845    }
846
847    fn call(&mut self, request: Frame) -> Self::Future {
848        let client = self.clone();
849        Box::pin(async move {
850            if let Some(queue) = &client.offline_queue {
851                match queue.before_execute(false) {
852                    QueueDecision::Execute => {}
853                    QueueDecision::Reject { error, campaign } => {
854                        if let Some(campaign) = campaign {
855                            client.spawn_reconnect_campaign(queue, campaign);
856                        }
857                        return Err(error.into_redis_error());
858                    }
859                    QueueDecision::Wait { .. } => {
860                        unreachable!("raw frames are never admitted to the offline queue")
861                    }
862                }
863            }
864
865            let mut connection = client.conn.lock().await;
866
867            if let Some(queue) = &client.offline_queue {
868                match queue.before_execute(false) {
869                    QueueDecision::Execute => {}
870                    QueueDecision::Reject { error, campaign } => {
871                        drop(connection);
872                        if let Some(campaign) = campaign {
873                            client.spawn_reconnect_campaign(queue, campaign);
874                        }
875                        return Err(error.into_redis_error());
876                    }
877                    QueueDecision::Wait { .. } => {
878                        unreachable!("raw frames are never admitted to the offline queue")
879                    }
880                }
881            }
882
883            let mut alignment = client
884                .offline_queue
885                .as_ref()
886                .map(|queue| ProtocolAlignmentGuard::new(&client, queue));
887            let result =
888                connection
889                    .execute_pipeline(vec![request])
890                    .await
891                    .and_then(|mut responses| {
892                        responses.pop().ok_or(RedisError::UnexpectedResponse {
893                            expected: "one pipeline response",
894                            actual: "empty response".to_string(),
895                        })
896                    });
897            if let Some(alignment) = &mut alignment {
898                alignment.disarm();
899            }
900
901            if matches!(&result, Err(error) if error.is_connection_error()) {
902                if let Err(error) = &result {
903                    client.publish_disconnect(error);
904                }
905                drop(connection);
906                if let Some(queue) = &client.offline_queue {
907                    match queue.after_connection_error(false) {
908                        QueueDecision::Reject { campaign, .. } => {
909                            if let Some(campaign) = campaign {
910                                client.spawn_reconnect_campaign(queue, campaign);
911                            }
912                        }
913                        QueueDecision::Execute | QueueDecision::Wait { .. } => {
914                            unreachable!("raw frames are never admitted to the offline queue")
915                        }
916                    }
917                } else {
918                    client.reconnect().await;
919                }
920            }
921            result
922        })
923    }
924}
925
926/// A dropped wire future leaves it unknown whether Redis will still send a
927/// response. Quarantine that connection synchronously on cancellation so the
928/// next queue ticket cannot consume the abandoned response.
929struct ProtocolAlignmentGuard<'a> {
930    client: &'a ResilientRedisClient,
931    queue: &'a OfflineQueueHandle,
932    armed: bool,
933}
934
935impl<'a> ProtocolAlignmentGuard<'a> {
936    fn new(client: &'a ResilientRedisClient, queue: &'a OfflineQueueHandle) -> Self {
937        Self {
938            client,
939            queue,
940            armed: true,
941        }
942    }
943
944    fn disarm(&mut self) {
945        self.armed = false;
946    }
947}
948
949impl Drop for ProtocolAlignmentGuard<'_> {
950    fn drop(&mut self) {
951        if !self.armed {
952            return;
953        }
954
955        self.client
956            .publish_disconnect(&RedisError::ConnectionClosed);
957        if let Some(campaign) = self.queue.taint_connection() {
958            self.client.spawn_reconnect_campaign(self.queue, campaign);
959        }
960    }
961}
962
963/// Execute a typed command while retaining the command value for a possible
964/// idempotent replay. This mirrors [`RedisConnection::execute`] while using a
965/// one-frame pipeline so serialization does not consume `cmd`.
966async fn execute_retained<Cmd: Command>(
967    conn: &mut RedisConnection,
968    cmd: Cmd,
969    frame: Frame,
970) -> (Cmd, Result<Cmd::Response, RedisError>) {
971    let result = match conn.execute_pipeline(vec![frame]).await {
972        Ok(mut responses) => match responses.pop() {
973            Some(Frame::Error(error)) => Err(RedisError::Redis(
974                String::from_utf8_lossy(&error).into_owned(),
975            )),
976            Some(response) => cmd.parse_response(response),
977            None => Err(RedisError::UnexpectedResponse {
978                expected: "one command response",
979                actual: "empty response".to_string(),
980            }),
981        },
982        Err(error) => Err(error),
983    };
984    (cmd, result)
985}
986
987/// Queue state shared by all client clones.
988struct OfflineQueue {
989    capacity: usize,
990    max_replay_attempts: usize,
991    owners: AtomicUsize,
992    state: StdMutex<OfflineQueueState>,
993    changed: Notify,
994    closed_tx: watch::Sender<bool>,
995}
996
997struct OfflineQueueState {
998    phase: OfflinePhase,
999    tickets: VecDeque<u64>,
1000    next_ticket: u64,
1001    next_campaign: u64,
1002    last_reconnect_failure: Option<ReconnectFailure>,
1003}
1004
1005#[derive(Debug, Clone)]
1006struct ReconnectFailure {
1007    attempts: usize,
1008    last_error: Arc<RedisError>,
1009}
1010
1011impl ReconnectFailure {
1012    fn rejection(&self) -> QueueRejection {
1013        QueueRejection::ReconnectFailed {
1014            attempts: self.attempts,
1015            last_error: Arc::clone(&self.last_error),
1016        }
1017    }
1018}
1019
1020#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1021enum OfflinePhase {
1022    Connected,
1023    Reconnecting { campaign: u64 },
1024    Draining,
1025    Failed { attempts: usize },
1026    Closed,
1027}
1028
1029/// An owner-counted handle. Reconnect tasks use `Weak<OfflineQueue>` rather
1030/// than cloning this handle, so dropping the final client clone closes the
1031/// queue and cancels a hanging factory future.
1032struct OfflineQueueHandle {
1033    inner: Arc<OfflineQueue>,
1034}
1035
1036impl OfflineQueueHandle {
1037    fn new(config: OfflineQueueConfig) -> Self {
1038        let (closed_tx, _) = watch::channel(false);
1039        Self {
1040            inner: Arc::new(OfflineQueue {
1041                capacity: config.capacity,
1042                max_replay_attempts: config.max_replay_attempts,
1043                owners: AtomicUsize::new(1),
1044                state: StdMutex::new(OfflineQueueState {
1045                    phase: OfflinePhase::Connected,
1046                    tickets: VecDeque::new(),
1047                    next_ticket: 0,
1048                    next_campaign: 0,
1049                    last_reconnect_failure: None,
1050                }),
1051                changed: Notify::new(),
1052                closed_tx,
1053            }),
1054        }
1055    }
1056
1057    fn state(&self) -> std::sync::MutexGuard<'_, OfflineQueueState> {
1058        self.inner
1059            .state
1060            .lock()
1061            .expect("offline queue state mutex poisoned")
1062    }
1063
1064    fn depth(&self) -> usize {
1065        self.state().tickets.len()
1066    }
1067
1068    fn is_reconnecting(&self) -> bool {
1069        matches!(self.state().phase, OfflinePhase::Reconnecting { .. })
1070    }
1071
1072    fn max_replay_attempts(&self) -> usize {
1073        self.inner.max_replay_attempts
1074    }
1075
1076    fn before_execute(&self, idempotent: bool) -> QueueDecision {
1077        let mut state = self.state();
1078        match state.phase {
1079            OfflinePhase::Connected => QueueDecision::Execute,
1080            OfflinePhase::Reconnecting { .. } | OfflinePhase::Draining => {
1081                if idempotent {
1082                    self.enqueue_locked(&mut state, None)
1083                } else {
1084                    QueueDecision::reject(QueueRejection::ConnectionClosed, None)
1085                }
1086            }
1087            OfflinePhase::Failed { attempts } if !state.tickets.is_empty() => {
1088                if idempotent {
1089                    let rejection = state.last_reconnect_failure.as_ref().map_or_else(
1090                        || QueueRejection::ReconnectFailed {
1091                            attempts,
1092                            last_error: Arc::new(RedisError::ConnectionClosed),
1093                        },
1094                        ReconnectFailure::rejection,
1095                    );
1096                    QueueDecision::reject(rejection, None)
1097                } else {
1098                    QueueDecision::reject(QueueRejection::ConnectionClosed, None)
1099                }
1100            }
1101            OfflinePhase::Failed { .. } => {
1102                let campaign = Self::begin_campaign_locked(&mut state);
1103                if idempotent {
1104                    self.enqueue_locked(&mut state, Some(campaign))
1105                } else {
1106                    QueueDecision::reject(QueueRejection::ConnectionClosed, Some(campaign))
1107                }
1108            }
1109            OfflinePhase::Closed => QueueDecision::reject(QueueRejection::ConnectionClosed, None),
1110        }
1111    }
1112
1113    fn after_connection_error(&self, idempotent: bool) -> QueueDecision {
1114        let mut state = self.state();
1115        let campaign = match state.phase {
1116            OfflinePhase::Connected | OfflinePhase::Draining | OfflinePhase::Failed { .. } => {
1117                Some(Self::begin_campaign_locked(&mut state))
1118            }
1119            OfflinePhase::Reconnecting { .. } => None,
1120            OfflinePhase::Closed => {
1121                return QueueDecision::reject(QueueRejection::ConnectionClosed, None);
1122            }
1123        };
1124
1125        if idempotent {
1126            self.enqueue_locked(&mut state, campaign)
1127        } else {
1128            QueueDecision::reject(QueueRejection::ConnectionClosed, campaign)
1129        }
1130    }
1131
1132    fn enqueue_locked(
1133        &self,
1134        state: &mut OfflineQueueState,
1135        campaign: Option<u64>,
1136    ) -> QueueDecision {
1137        if state.tickets.len() >= self.inner.capacity {
1138            return QueueDecision::reject(QueueRejection::Full, campaign);
1139        }
1140        let ticket = state.next_ticket;
1141        state.next_ticket = state.next_ticket.wrapping_add(1);
1142        state.tickets.push_back(ticket);
1143        QueueDecision::Wait {
1144            permit: QueuePermit {
1145                queue: Arc::clone(&self.inner),
1146                ticket,
1147            },
1148            campaign,
1149        }
1150    }
1151
1152    fn begin_campaign_locked(state: &mut OfflineQueueState) -> u64 {
1153        let campaign = state.next_campaign;
1154        state.next_campaign = state.next_campaign.wrapping_add(1);
1155        state.phase = OfflinePhase::Reconnecting { campaign };
1156        state.last_reconnect_failure = None;
1157        campaign
1158    }
1159
1160    fn reconnect_existing(&self, permit: &QueuePermit) -> Option<u64> {
1161        let mut state = self.state();
1162        if matches!(state.phase, OfflinePhase::Closed) {
1163            return None;
1164        }
1165        if matches!(state.phase, OfflinePhase::Reconnecting { .. }) {
1166            return None;
1167        }
1168        debug_assert_eq!(state.tickets.front(), Some(&permit.ticket));
1169        Some(Self::begin_campaign_locked(&mut state))
1170    }
1171
1172    /// Mark a possibly half-consumed connection unusable after the command
1173    /// future is canceled while an exchange is on the wire.
1174    fn taint_connection(&self) -> Option<u64> {
1175        let mut state = self.state();
1176        match state.phase {
1177            OfflinePhase::Connected | OfflinePhase::Draining | OfflinePhase::Failed { .. } => {
1178                Some(Self::begin_campaign_locked(&mut state))
1179            }
1180            OfflinePhase::Reconnecting { .. } | OfflinePhase::Closed => None,
1181        }
1182    }
1183}
1184
1185impl Clone for OfflineQueueHandle {
1186    fn clone(&self) -> Self {
1187        self.inner.owners.fetch_add(1, Ordering::Relaxed);
1188        Self {
1189            inner: Arc::clone(&self.inner),
1190        }
1191    }
1192}
1193
1194impl Drop for OfflineQueueHandle {
1195    fn drop(&mut self) {
1196        if self.inner.owners.fetch_sub(1, Ordering::AcqRel) == 1 {
1197            self.inner.close();
1198        }
1199    }
1200}
1201
1202impl OfflineQueue {
1203    fn state(&self) -> std::sync::MutexGuard<'_, OfflineQueueState> {
1204        self.state
1205            .lock()
1206            .expect("offline queue state mutex poisoned")
1207    }
1208
1209    fn close(&self) {
1210        self.state().phase = OfflinePhase::Closed;
1211        self.closed_tx.send_replace(true);
1212        self.changed.notify_waiters();
1213    }
1214
1215    fn campaign_is_active(&self, campaign: u64) -> bool {
1216        matches!(
1217            self.state().phase,
1218            OfflinePhase::Reconnecting { campaign: active } if active == campaign
1219        )
1220    }
1221
1222    fn finish_reconnect(&self, campaign: u64, failure: Option<ReconnectFailure>) {
1223        let mut state = self.state();
1224        if !matches!(
1225            state.phase,
1226            OfflinePhase::Reconnecting { campaign: active } if active == campaign
1227        ) {
1228            return;
1229        }
1230        state.phase = match &failure {
1231            Some(failure) => OfflinePhase::Failed {
1232                attempts: failure.attempts,
1233            },
1234            None if state.tickets.is_empty() => OfflinePhase::Connected,
1235            None => OfflinePhase::Draining,
1236        };
1237        state.last_reconnect_failure = failure;
1238        drop(state);
1239        self.changed.notify_waiters();
1240    }
1241}
1242
1243enum QueueDecision {
1244    Execute,
1245    Wait {
1246        permit: QueuePermit,
1247        campaign: Option<u64>,
1248    },
1249    Reject {
1250        error: QueueRejection,
1251        campaign: Option<u64>,
1252    },
1253}
1254
1255impl QueueDecision {
1256    fn reject(error: QueueRejection, campaign: Option<u64>) -> Self {
1257        Self::Reject { error, campaign }
1258    }
1259}
1260
1261#[derive(Debug, Clone)]
1262enum QueueRejection {
1263    ConnectionClosed,
1264    Full,
1265    ReconnectFailed {
1266        attempts: usize,
1267        last_error: Arc<RedisError>,
1268    },
1269}
1270
1271impl QueueRejection {
1272    fn into_redis_error(self) -> RedisError {
1273        match self {
1274            Self::ConnectionClosed => RedisError::ConnectionClosed,
1275            Self::Full => RedisError::QueueFull,
1276            Self::ReconnectFailed {
1277                attempts,
1278                last_error,
1279            } => RedisError::ReconnectFailed {
1280                attempts,
1281                last_error,
1282            },
1283        }
1284    }
1285}
1286
1287struct QueuePermit {
1288    queue: Arc<OfflineQueue>,
1289    ticket: u64,
1290}
1291
1292impl QueuePermit {
1293    async fn wait_for_turn(&self) -> Result<(), RedisError> {
1294        loop {
1295            let notified = self.queue.changed.notified();
1296            match self.check_turn() {
1297                TurnState::Ready => return Ok(()),
1298                TurnState::Wait => notified.await,
1299                TurnState::Rejected(error) => return Err(error.into_redis_error()),
1300            }
1301        }
1302    }
1303
1304    fn check_turn(&self) -> TurnState {
1305        let state = self.queue.state();
1306        match state.phase {
1307            OfflinePhase::Connected | OfflinePhase::Draining
1308                if state.tickets.front() == Some(&self.ticket) =>
1309            {
1310                TurnState::Ready
1311            }
1312            OfflinePhase::Failed { attempts } => {
1313                let rejection = state.last_reconnect_failure.as_ref().map_or_else(
1314                    || QueueRejection::ReconnectFailed {
1315                        attempts,
1316                        last_error: Arc::new(RedisError::ConnectionClosed),
1317                    },
1318                    ReconnectFailure::rejection,
1319                );
1320                TurnState::Rejected(rejection)
1321            }
1322            OfflinePhase::Closed => TurnState::Rejected(QueueRejection::ConnectionClosed),
1323            _ => TurnState::Wait,
1324        }
1325    }
1326}
1327
1328impl Drop for QueuePermit {
1329    fn drop(&mut self) {
1330        let mut state = self.queue.state();
1331        let was_front = state.tickets.front() == Some(&self.ticket);
1332        if let Some(index) = state
1333            .tickets
1334            .iter()
1335            .position(|ticket| *ticket == self.ticket)
1336        {
1337            state.tickets.remove(index);
1338        }
1339        if state.tickets.is_empty() && matches!(state.phase, OfflinePhase::Draining) {
1340            state.phase = OfflinePhase::Connected;
1341        }
1342        drop(state);
1343        if was_front {
1344            self.queue.changed.notify_waiters();
1345        }
1346    }
1347}
1348
1349enum TurnState {
1350    Ready,
1351    Wait,
1352    Rejected(QueueRejection),
1353}
1354
1355/// Connect via the factory, bounding the attempt by `connect_timeout` if one is
1356/// configured so a black-holed connect cannot hang the reconnect loop forever.
1357async fn connect_with_timeout(
1358    factory: &dyn ConnectionFactory,
1359    connect_timeout: Option<Duration>,
1360) -> Result<RedisConnection, RedisError> {
1361    match connect_timeout {
1362        Some(t) => tokio::time::timeout(t, factory.connect())
1363            .await
1364            .map_err(|_| RedisError::ConnectTimeout)?,
1365        None => factory.connect().await,
1366    }
1367}
1368
1369/// Run a reconnect campaign, applying the configured exponential delay before
1370/// every attempt (including `base_delay` before the first). The campaign makes
1371/// one attempt plus up to `max_retries` additional retries and returns the new
1372/// connection, or `None` once that budget is exhausted.
1373async fn reconnect_campaign(
1374    factory: &dyn ConnectionFactory,
1375    config: &ReconnectConfig,
1376    lifecycle: Option<&ResilientLifecycleControl>,
1377) -> Option<ReconnectSuccess> {
1378    let max = config.max_retries.unwrap_or(usize::MAX);
1379    let started = Instant::now();
1380    for attempt in 0..=max {
1381        let delay = config.delay_for_attempt(attempt);
1382        tracing::warn!(attempt, delay = ?delay, "redis: backing off before reconnect");
1383        if let Some(lifecycle) = lifecycle
1384            && !lifecycle.publish(ConnectionEvent::ReconnectAttempt {
1385                attempt: attempt + 1,
1386                delay,
1387            })
1388        {
1389            return None;
1390        }
1391        if !delay.is_zero() {
1392            tokio::time::sleep(delay).await;
1393        }
1394        match connect_with_timeout(factory, config.connect_timeout).await {
1395            Ok(conn) => {
1396                tracing::info!(attempt, "redis: reconnected successfully");
1397                return Some(ReconnectSuccess {
1398                    connection: conn,
1399                    attempts: attempt + 1,
1400                    elapsed: started.elapsed(),
1401                });
1402            }
1403            Err(e) => {
1404                tracing::warn!(attempt, error = %e, "redis: reconnect attempt failed");
1405                if let Some(lifecycle) = lifecycle
1406                    && !lifecycle.publish_with(|| ConnectionEvent::ReconnectFailed {
1407                        attempt: attempt + 1,
1408                        error: Arc::from(e.to_string()),
1409                    })
1410                {
1411                    return None;
1412                }
1413            }
1414        }
1415    }
1416    if let Some(lifecycle) = lifecycle {
1417        lifecycle.publish(ConnectionEvent::ReconnectExhausted {
1418            attempts: max.saturating_add(1),
1419        });
1420    }
1421    None
1422}
1423
1424struct ReconnectSuccess {
1425    connection: RedisConnection,
1426    attempts: usize,
1427    elapsed: Duration,
1428}
1429
1430enum CancellableReconnect {
1431    Connected {
1432        connection: RedisConnection,
1433        attempts: usize,
1434        elapsed: Duration,
1435    },
1436    Failed {
1437        attempts: usize,
1438        last_error: RedisError,
1439    },
1440    Closed,
1441}
1442
1443/// Queue-enabled reconnects run independently of the caller that discovered
1444/// the failure. Watching queue closure makes that detached task cancellation
1445/// safe in both directions: canceling one command does not cancel everybody's
1446/// reconnect, while dropping every client clone does cancel a hanging factory.
1447async fn reconnect_campaign_until_closed(
1448    factory: &dyn ConnectionFactory,
1449    config: &ReconnectConfig,
1450    closed: &mut watch::Receiver<bool>,
1451    lifecycle: Option<&ResilientLifecycleControl>,
1452) -> CancellableReconnect {
1453    let max = config.max_retries.unwrap_or(usize::MAX);
1454    let started = Instant::now();
1455    let mut last_error = None;
1456    for attempt in 0..=max {
1457        if *closed.borrow() {
1458            return CancellableReconnect::Closed;
1459        }
1460
1461        let delay = config.delay_for_attempt(attempt);
1462        tracing::warn!(attempt, delay = ?delay, "redis: backing off before reconnect");
1463        if let Some(lifecycle) = lifecycle
1464            && !lifecycle.publish(ConnectionEvent::ReconnectAttempt {
1465                attempt: attempt + 1,
1466                delay,
1467            })
1468        {
1469            return CancellableReconnect::Closed;
1470        }
1471        if !delay.is_zero() {
1472            tokio::select! {
1473                // Once shutdown and delay are both ready, shutdown wins.
1474                biased;
1475                result = closed.changed() => {
1476                    let _ = result;
1477                    return CancellableReconnect::Closed;
1478                }
1479                () = tokio::time::sleep(delay) => {}
1480            }
1481        }
1482
1483        let connect = connect_with_timeout(factory, config.connect_timeout);
1484        tokio::pin!(connect);
1485        let result = tokio::select! {
1486            // Do not accept a replacement after terminal close when both
1487            // branches become ready in the same scheduler turn.
1488            biased;
1489            result = closed.changed() => {
1490                let _ = result;
1491                return CancellableReconnect::Closed;
1492            }
1493            result = &mut connect => result,
1494        };
1495
1496        match result {
1497            Ok(conn) => {
1498                tracing::info!(attempt, "redis: reconnected successfully");
1499                return CancellableReconnect::Connected {
1500                    connection: conn,
1501                    attempts: attempt + 1,
1502                    elapsed: started.elapsed(),
1503                };
1504            }
1505            Err(error) => {
1506                tracing::warn!(attempt, error = %error, "redis: reconnect attempt failed");
1507                if let Some(lifecycle) = lifecycle
1508                    && !lifecycle.publish_with(|| ConnectionEvent::ReconnectFailed {
1509                        attempt: attempt + 1,
1510                        error: Arc::from(error.to_string()),
1511                    })
1512                {
1513                    return CancellableReconnect::Closed;
1514                }
1515                last_error = Some(error);
1516            }
1517        }
1518    }
1519
1520    let attempts = max.saturating_add(1);
1521    if let Some(lifecycle) = lifecycle
1522        && !lifecycle.publish(ConnectionEvent::ReconnectExhausted { attempts })
1523    {
1524        return CancellableReconnect::Closed;
1525    }
1526    CancellableReconnect::Failed {
1527        attempts,
1528        last_error: last_error.unwrap_or(RedisError::ConnectionClosed),
1529    }
1530}
1531
1532/// Single-flight coordinator for reconnects shared across clones.
1533///
1534/// A generation counter, bumped once per successful reconnect, lets a clone
1535/// that was waiting on the gate detect that the connection was already replaced
1536/// and skip its own attempt.
1537struct ReconnectGate {
1538    lock: Mutex<()>,
1539    generation: AtomicU64,
1540    reconnecting: std::sync::atomic::AtomicBool,
1541}
1542
1543impl ReconnectGate {
1544    fn new() -> Self {
1545        Self {
1546            lock: Mutex::new(()),
1547            generation: AtomicU64::new(0),
1548            reconnecting: std::sync::atomic::AtomicBool::new(false),
1549        }
1550    }
1551
1552    /// The current reconnect generation.
1553    fn generation(&self) -> u64 {
1554        self.generation.load(Ordering::Acquire)
1555    }
1556
1557    /// Take the gate; only one clone holds it at a time, the rest wait.
1558    async fn enter(&self) -> tokio::sync::MutexGuard<'_, ()> {
1559        self.lock.lock().await
1560    }
1561
1562    /// Advance the generation after a successful reconnect.
1563    fn mark_reconnected(&self) {
1564        self.generation.fetch_add(1, Ordering::AcqRel);
1565    }
1566
1567    fn is_reconnecting(&self) -> bool {
1568        self.reconnecting.load(Ordering::Acquire)
1569    }
1570
1571    fn mark_reconnecting(&self) -> ActiveReconnect<'_> {
1572        self.reconnecting.store(true, Ordering::Release);
1573        ActiveReconnect(&self.reconnecting)
1574    }
1575}
1576
1577struct ActiveReconnect<'a>(&'a std::sync::atomic::AtomicBool);
1578
1579impl Drop for ActiveReconnect<'_> {
1580    fn drop(&mut self) {
1581        self.0.store(false, Ordering::Release);
1582    }
1583}
1584
1585#[cfg(test)]
1586mod tests {
1587    use super::*;
1588    use bytes::Bytes;
1589    use futures::{SinkExt, StreamExt};
1590    use redis_tower_core::{RedisStream, RespCodec, WithDeadline};
1591    use redis_tower_protocol::helpers::{array, bulk};
1592    use std::future::Future;
1593    use std::pin::Pin;
1594    use std::sync::atomic::AtomicUsize;
1595    use std::time::Instant;
1596    use tokio::io::AsyncReadExt;
1597    use tokio::net::{TcpListener, TcpStream};
1598    use tokio::sync::{Semaphore, oneshot};
1599    use tokio_util::codec::Framed;
1600
1601    type ConnFuture = Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>>;
1602
1603    /// Factory whose `connect()` always fails immediately, counting calls.
1604    struct FailingFactory {
1605        calls: Arc<AtomicUsize>,
1606    }
1607
1608    impl ConnectionFactory for FailingFactory {
1609        fn connect(&self) -> ConnFuture {
1610            self.calls.fetch_add(1, Ordering::SeqCst);
1611            Box::pin(async { Err(RedisError::ConnectionClosed) })
1612        }
1613    }
1614
1615    /// Factory whose `connect()` never resolves, counting calls.
1616    struct HangingFactory {
1617        calls: Arc<AtomicUsize>,
1618    }
1619
1620    impl ConnectionFactory for HangingFactory {
1621        fn connect(&self) -> ConnFuture {
1622            self.calls.fetch_add(1, Ordering::SeqCst);
1623            Box::pin(std::future::pending())
1624        }
1625    }
1626
1627    /// The reconnect future becomes ready only when the test flips `ready`;
1628    /// it intentionally stores no waker, so queue closure is the wake that
1629    /// makes both select branches ready in the same poll.
1630    struct ReadyOnCloseFactory {
1631        calls: Arc<AtomicUsize>,
1632        initial: StdMutex<Option<RedisConnection>>,
1633        ready: Arc<AtomicBool>,
1634    }
1635
1636    impl ConnectionFactory for ReadyOnCloseFactory {
1637        fn connect(&self) -> ConnFuture {
1638            let call = self.calls.fetch_add(1, Ordering::SeqCst);
1639            if call == 0 {
1640                let connection = self
1641                    .initial
1642                    .lock()
1643                    .expect("initial connection mutex poisoned")
1644                    .take();
1645                return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1646            }
1647
1648            let ready = Arc::clone(&self.ready);
1649            Box::pin(std::future::poll_fn(move |_cx| {
1650                if ready.load(Ordering::Acquire) {
1651                    std::task::Poll::Ready(Err(RedisError::ConnectionClosed))
1652                } else {
1653                    std::task::Poll::Pending
1654                }
1655            }))
1656        }
1657    }
1658
1659    struct EchoCommand {
1660        value: Bytes,
1661        idempotent: bool,
1662    }
1663
1664    impl EchoCommand {
1665        fn new(value: &'static str, idempotent: bool) -> Self {
1666            Self {
1667                value: Bytes::from_static(value.as_bytes()),
1668                idempotent,
1669            }
1670        }
1671    }
1672
1673    impl Command for EchoCommand {
1674        type Response = Bytes;
1675
1676        fn to_frame(&self) -> Frame {
1677            array(vec![bulk("ECHO"), bulk(self.value.clone())])
1678        }
1679
1680        fn parse_response(&self, frame: Frame) -> Result<Self::Response, RedisError> {
1681            match frame {
1682                Frame::BulkString(Some(value)) => Ok(value),
1683                other => Err(RedisError::UnexpectedResponse {
1684                    expected: "bulk string",
1685                    actual: format!("{other:?}"),
1686                }),
1687            }
1688        }
1689
1690        fn name(&self) -> &str {
1691            "ECHO"
1692        }
1693
1694        fn idempotent(&self) -> bool {
1695            self.idempotent
1696        }
1697    }
1698
1699    /// Factory with an immediate initial connection and a replacement held
1700    /// behind a semaphore so tests can fill and inspect the offline queue.
1701    struct ControlledFactory {
1702        calls: Arc<AtomicUsize>,
1703        initial: StdMutex<Option<RedisConnection>>,
1704        replacement: Arc<StdMutex<Option<RedisConnection>>>,
1705        release_reconnect: Arc<Semaphore>,
1706    }
1707
1708    impl ConnectionFactory for ControlledFactory {
1709        fn connect(&self) -> ConnFuture {
1710            let call = self.calls.fetch_add(1, Ordering::SeqCst);
1711            if call == 0 {
1712                let connection = self
1713                    .initial
1714                    .lock()
1715                    .expect("initial connection mutex poisoned")
1716                    .take();
1717                return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1718            }
1719
1720            let replacement = Arc::clone(&self.replacement);
1721            let release = Arc::clone(&self.release_reconnect);
1722            Box::pin(async move {
1723                let permit = release
1724                    .acquire_owned()
1725                    .await
1726                    .map_err(|_| RedisError::ConnectionClosed)?;
1727                permit.forget();
1728                replacement
1729                    .lock()
1730                    .expect("replacement connection mutex poisoned")
1731                    .take()
1732                    .ok_or(RedisError::ConnectionClosed)
1733            })
1734        }
1735    }
1736
1737    /// Factory that yields several independently gated replacement sockets.
1738    /// This lets cancellation tests prove that a late response on one socket
1739    /// can never be consumed after the client advances to the next ticket.
1740    struct SequencedFactory {
1741        calls: Arc<AtomicUsize>,
1742        initial: StdMutex<Option<RedisConnection>>,
1743        replacements: Arc<StdMutex<VecDeque<RedisConnection>>>,
1744        release_reconnect: Arc<Semaphore>,
1745    }
1746
1747    impl ConnectionFactory for SequencedFactory {
1748        fn connect(&self) -> ConnFuture {
1749            let call = self.calls.fetch_add(1, Ordering::SeqCst);
1750            if call == 0 {
1751                let connection = self
1752                    .initial
1753                    .lock()
1754                    .expect("initial connection mutex poisoned")
1755                    .take();
1756                return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1757            }
1758
1759            let replacements = Arc::clone(&self.replacements);
1760            let release = Arc::clone(&self.release_reconnect);
1761            Box::pin(async move {
1762                let permit = release
1763                    .acquire_owned()
1764                    .await
1765                    .map_err(|_| RedisError::ConnectionClosed)?;
1766                permit.forget();
1767                replacements
1768                    .lock()
1769                    .expect("replacement connections mutex poisoned")
1770                    .pop_front()
1771                    .ok_or(RedisError::ConnectionClosed)
1772            })
1773        }
1774    }
1775
1776    async fn connection_pair() -> (RedisConnection, TcpStream) {
1777        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1778        let addr = listener.local_addr().unwrap();
1779        let (client, accepted) = tokio::join!(TcpStream::connect(addr), listener.accept());
1780        let client = client.unwrap();
1781        let (server, _) = accepted.unwrap();
1782        (
1783            RedisConnection::from_stream(RedisStream::Tcp(client)),
1784            server,
1785        )
1786    }
1787
1788    fn close_on_first_request(mut server: TcpStream) -> oneshot::Receiver<()> {
1789        let (seen_tx, seen_rx) = oneshot::channel();
1790        tokio::spawn(async move {
1791            let mut bytes = [0; 1024];
1792            let read = server.read(&mut bytes).await.unwrap();
1793            assert!(read > 0, "mock Redis did not receive the first command");
1794            let _ = seen_tx.send(());
1795            // Dropping the socket without a response creates the connection
1796            // error that starts the reconnect campaign.
1797        });
1798        seen_rx
1799    }
1800
1801    fn hold_after_first_request(
1802        mut server: TcpStream,
1803    ) -> (oneshot::Receiver<()>, tokio::task::JoinHandle<()>) {
1804        let (seen_tx, seen_rx) = oneshot::channel();
1805        let task = tokio::spawn(async move {
1806            let mut bytes = [0; 1024];
1807            let read = server.read(&mut bytes).await.unwrap();
1808            assert!(read > 0, "mock Redis did not receive the held command");
1809            let _ = seen_tx.send(());
1810            std::future::pending::<()>().await;
1811        });
1812        (seen_rx, task)
1813    }
1814
1815    fn delayed_response_after_first_request(
1816        server: TcpStream,
1817        response: Bytes,
1818    ) -> (
1819        oneshot::Receiver<()>,
1820        oneshot::Sender<()>,
1821        tokio::task::JoinHandle<()>,
1822    ) {
1823        let (seen_tx, seen_rx) = oneshot::channel();
1824        let (respond_tx, respond_rx) = oneshot::channel();
1825        let task = tokio::spawn(async move {
1826            let mut framed = Framed::new(server, RespCodec::new());
1827            framed
1828                .next()
1829                .await
1830                .expect("mock Redis connection closed before held request")
1831                .expect("mock Redis received invalid RESP");
1832            let _ = seen_tx.send(());
1833            respond_rx.await.expect("late response signal dropped");
1834            framed
1835                .send(Frame::BulkString(Some(response)))
1836                .await
1837                .expect("late response should reach the quarantined socket");
1838        });
1839        (seen_rx, respond_tx, task)
1840    }
1841
1842    fn echo_server(server: TcpStream, expected: usize) -> tokio::task::JoinHandle<Vec<Bytes>> {
1843        tokio::spawn(async move {
1844            let mut framed = Framed::new(server, RespCodec::new());
1845            let mut seen = Vec::with_capacity(expected);
1846            while seen.len() < expected {
1847                let request = framed
1848                    .next()
1849                    .await
1850                    .expect("mock Redis connection closed early")
1851                    .expect("mock Redis received invalid RESP");
1852                let value = match request {
1853                    Frame::Array(Some(parts)) => match parts.get(1) {
1854                        Some(Frame::BulkString(Some(value))) => value.clone(),
1855                        other => panic!("expected ECHO bulk argument, got {other:?}"),
1856                    },
1857                    other => panic!("expected command array, got {other:?}"),
1858                };
1859                seen.push(value.clone());
1860                framed.send(Frame::BulkString(Some(value))).await.unwrap();
1861            }
1862            seen
1863        })
1864    }
1865
1866    async fn wait_for_depth(client: &ResilientRedisClient, expected: usize) {
1867        tokio::time::timeout(Duration::from_secs(1), async {
1868            loop {
1869                if client.offline_queue_depth() == expected {
1870                    return;
1871                }
1872                tokio::task::yield_now().await;
1873            }
1874        })
1875        .await
1876        .unwrap_or_else(|_| panic!("offline queue never reached depth {expected}"));
1877    }
1878
1879    async fn wait_for_calls(calls: &AtomicUsize, expected: usize) {
1880        tokio::time::timeout(Duration::from_secs(1), async {
1881            loop {
1882                if calls.load(Ordering::SeqCst) == expected {
1883                    return;
1884                }
1885                tokio::task::yield_now().await;
1886            }
1887        })
1888        .await
1889        .unwrap_or_else(|_| panic!("factory never reached {expected} calls"));
1890    }
1891
1892    fn config(
1893        max_retries: usize,
1894        base: Duration,
1895        connect_timeout: Option<Duration>,
1896    ) -> ReconnectConfig {
1897        ReconnectConfig {
1898            max_retries: Some(max_retries),
1899            base_delay: base,
1900            jitter: false,
1901            connect_timeout,
1902            ..Default::default()
1903        }
1904    }
1905
1906    // -- defect 1: connect_with_timeout --
1907
1908    #[tokio::test]
1909    async fn connect_with_timeout_propagates_factory_error() {
1910        let calls = Arc::new(AtomicUsize::new(0));
1911        let factory = FailingFactory {
1912            calls: calls.clone(),
1913        };
1914        // RedisConnection is not Debug, so match rather than unwrap_err.
1915        let result = connect_with_timeout(&factory, None).await;
1916        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
1917        assert_eq!(calls.load(Ordering::SeqCst), 1);
1918    }
1919
1920    #[tokio::test]
1921    async fn connect_with_timeout_times_out_a_hanging_connect() {
1922        let calls = Arc::new(AtomicUsize::new(0));
1923        let factory = HangingFactory {
1924            calls: calls.clone(),
1925        };
1926        let start = Instant::now();
1927        let result = connect_with_timeout(&factory, Some(Duration::from_millis(30))).await;
1928        // Without the timeout this would hang forever.
1929        assert!(matches!(result, Err(RedisError::ConnectTimeout)));
1930        assert!(start.elapsed() < Duration::from_secs(2));
1931        assert_eq!(calls.load(Ordering::SeqCst), 1);
1932    }
1933
1934    // -- reconnect delay semantics --
1935
1936    #[tokio::test]
1937    async fn reconnect_campaign_applies_base_delay_before_first_attempt() {
1938        let calls = Arc::new(AtomicUsize::new(0));
1939        let factory = FailingFactory {
1940            calls: calls.clone(),
1941        };
1942        let initial_delay = Duration::from_millis(20);
1943        let cfg = config(0, initial_delay, None);
1944        let events = ConnectionEventBus::new(2);
1945        let mut stream = events.subscribe();
1946        let lifecycle = ResilientLifecycleControl::new(events.clone());
1947        let mut campaign = Box::pin(reconnect_campaign(&factory, &cfg, Some(&lifecycle)));
1948
1949        assert!(futures::poll!(campaign.as_mut()).is_pending());
1950        assert_eq!(calls.load(Ordering::SeqCst), 0);
1951        assert_eq!(
1952            stream.recv().await.unwrap(),
1953            ConnectionEvent::ReconnectAttempt {
1954                attempt: 1,
1955                delay: initial_delay,
1956            }
1957        );
1958
1959        let result = tokio::time::timeout(Duration::from_secs(1), campaign)
1960            .await
1961            .expect("campaign should run after its initial delay");
1962        assert!(result.is_none());
1963        assert_eq!(calls.load(Ordering::SeqCst), 1);
1964    }
1965
1966    #[tokio::test]
1967    async fn reconnect_campaign_backs_off_between_retries() {
1968        let calls = Arc::new(AtomicUsize::new(0));
1969        let factory = FailingFactory {
1970            calls: calls.clone(),
1971        };
1972        // 3 attempts total (initial + 2 retries) with tiny backoff.
1973        let cfg = config(2, Duration::from_millis(1), None);
1974        let result = reconnect_campaign(&factory, &cfg, None).await;
1975        assert!(result.is_none());
1976        assert_eq!(calls.load(Ordering::SeqCst), 3);
1977    }
1978
1979    #[tokio::test]
1980    async fn reconnect_campaign_honors_connect_timeout_per_attempt() {
1981        let calls = Arc::new(AtomicUsize::new(0));
1982        let factory = HangingFactory {
1983            calls: calls.clone(),
1984        };
1985        let cfg = config(0, Duration::from_millis(1), Some(Duration::from_millis(30)));
1986        let result = reconnect_campaign(&factory, &cfg, None).await;
1987        assert!(result.is_none());
1988        assert_eq!(calls.load(Ordering::SeqCst), 1);
1989    }
1990
1991    #[tokio::test]
1992    async fn reconnect_campaign_publishes_failure_and_exhaustion_in_order() {
1993        let factory = FailingFactory {
1994            calls: Arc::new(AtomicUsize::new(0)),
1995        };
1996        let events = ConnectionEventBus::new(8);
1997        let mut stream = events.subscribe();
1998        let lifecycle = ResilientLifecycleControl::new(events.clone());
1999        let cfg = config(1, Duration::ZERO, None);
2000        assert!(
2001            reconnect_campaign(&factory, &cfg, Some(&lifecycle))
2002                .await
2003                .is_none()
2004        );
2005
2006        for attempt in 1..=2 {
2007            assert_eq!(
2008                stream.recv().await.unwrap(),
2009                ConnectionEvent::ReconnectAttempt {
2010                    attempt,
2011                    delay: Duration::ZERO,
2012                }
2013            );
2014            assert!(matches!(
2015                stream.recv().await.unwrap(),
2016                ConnectionEvent::ReconnectFailed {
2017                    attempt: failed,
2018                    ..
2019                } if failed == attempt
2020            ));
2021        }
2022        assert_eq!(
2023            stream.recv().await.unwrap(),
2024            ConnectionEvent::ReconnectExhausted { attempts: 2 }
2025        );
2026    }
2027
2028    #[tokio::test]
2029    async fn event_constructor_publishes_initial_connect_failure() {
2030        let events = ConnectionEventBus::new(2);
2031        let mut stream = events.subscribe();
2032        let result = ResilientRedisClient::with_config_and_events(
2033            FailingFactory {
2034                calls: Arc::new(AtomicUsize::new(0)),
2035            },
2036            config(0, Duration::ZERO, None),
2037            events,
2038        )
2039        .await;
2040        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
2041        assert!(matches!(
2042            stream.recv().await.unwrap(),
2043            ConnectionEvent::ConnectFailed { .. }
2044        ));
2045    }
2046
2047    #[tokio::test]
2048    async fn event_clients_publish_shutdown_once_when_final_clone_drops() {
2049        use crate::reconnect::ConnectionDisconnectReason;
2050
2051        for offline_queue in [false, true] {
2052            let (connection, _server) = connection_pair().await;
2053            let initial = Arc::new(StdMutex::new(Some(connection)));
2054            let factory = {
2055                let initial = Arc::clone(&initial);
2056                move || {
2057                    let connection = initial
2058                        .lock()
2059                        .expect("initial connection mutex poisoned")
2060                        .take();
2061                    async move { connection.ok_or(RedisError::ConnectionClosed) }
2062                }
2063            };
2064            let events = ConnectionEventBus::new(4);
2065            let mut stream = events.subscribe();
2066            let client = if offline_queue {
2067                ResilientRedisClient::with_config_and_offline_queue_and_events(
2068                    factory,
2069                    config(0, Duration::ZERO, None),
2070                    OfflineQueueConfig::new(1),
2071                    events.clone(),
2072                )
2073                .await
2074                .unwrap()
2075            } else {
2076                ResilientRedisClient::with_config_and_events(
2077                    factory,
2078                    config(0, Duration::ZERO, None),
2079                    events.clone(),
2080                )
2081                .await
2082                .unwrap()
2083            };
2084            assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2085
2086            let last = client.clone();
2087            drop(client);
2088            assert!(
2089                tokio::time::timeout(Duration::from_millis(10), stream.recv())
2090                    .await
2091                    .is_err(),
2092                "dropping a non-final clone must not emit shutdown"
2093            );
2094            drop(last);
2095            assert_eq!(
2096                stream.recv().await.unwrap(),
2097                ConnectionEvent::Disconnected {
2098                    reason: ConnectionDisconnectReason::Shutdown,
2099                }
2100            );
2101            assert!(
2102                tokio::time::timeout(Duration::from_millis(10), stream.recv())
2103                    .await
2104                    .is_err(),
2105                "final clone drop must emit shutdown exactly once"
2106            );
2107        }
2108    }
2109
2110    #[tokio::test]
2111    async fn lifecycle_gate_orders_in_progress_publication_before_shutdown() {
2112        let events = ConnectionEventBus::new(4);
2113        let mut stream = events.subscribe();
2114        let lease = ResilientClientLifecycle::new(events);
2115        let control = lease.control();
2116        let (entered_tx, entered_rx) = std::sync::mpsc::channel();
2117        let (release_tx, release_rx) = std::sync::mpsc::channel();
2118
2119        let publisher = std::thread::spawn(move || {
2120            control.publish_with(|| {
2121                entered_tx.send(()).unwrap();
2122                release_rx.recv().unwrap();
2123                ConnectionEvent::ReconnectFailed {
2124                    attempt: 1,
2125                    error: Arc::from("coordinated failure"),
2126                }
2127            })
2128        });
2129        entered_rx
2130            .recv_timeout(Duration::from_secs(1))
2131            .expect("publication never reached the between-check hook");
2132
2133        let (drop_started_tx, drop_started_rx) = std::sync::mpsc::channel();
2134        let dropper = std::thread::spawn(move || {
2135            drop_started_tx.send(()).unwrap();
2136            drop(lease);
2137        });
2138        drop_started_rx
2139            .recv_timeout(Duration::from_secs(1))
2140            .unwrap();
2141        release_tx.send(()).unwrap();
2142
2143        assert!(publisher.join().unwrap());
2144        dropper.join().unwrap();
2145        assert_eq!(
2146            stream.recv().await.unwrap(),
2147            ConnectionEvent::ReconnectFailed {
2148                attempt: 1,
2149                error: Arc::from("coordinated failure"),
2150            }
2151        );
2152        assert_eq!(
2153            stream.recv().await.unwrap(),
2154            ConnectionEvent::Disconnected {
2155                reason: ConnectionDisconnectReason::Shutdown,
2156            }
2157        );
2158    }
2159
2160    #[tokio::test]
2161    async fn final_clone_drop_publishes_shutdown_during_reported_outage() {
2162        let (initial, initial_server) = connection_pair().await;
2163        let initial_seen = close_on_first_request(initial_server);
2164        let (unused_replacement, _replacement_server) = connection_pair().await;
2165        let calls = Arc::new(AtomicUsize::new(0));
2166        let release = Arc::new(Semaphore::new(0));
2167        let factory = ControlledFactory {
2168            calls: Arc::clone(&calls),
2169            initial: StdMutex::new(Some(initial)),
2170            replacement: Arc::new(StdMutex::new(Some(unused_replacement))),
2171            release_reconnect: release,
2172        };
2173        let events = ConnectionEventBus::new(8);
2174        let mut stream = events.subscribe();
2175        let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2176            factory,
2177            config(0, Duration::ZERO, None),
2178            OfflineQueueConfig::new(1),
2179            events.clone(),
2180        )
2181        .await
2182        .unwrap();
2183        assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2184
2185        let command_client = client.clone();
2186        let command = tokio::spawn(async move {
2187            command_client
2188                .execute(EchoCommand::new("outage", true))
2189                .await
2190        });
2191        initial_seen.await.unwrap();
2192        assert!(matches!(
2193            stream.recv().await.unwrap(),
2194            ConnectionEvent::Disconnected {
2195                reason: ConnectionDisconnectReason::ConnectionError { .. }
2196            }
2197        ));
2198        assert_eq!(
2199            stream.recv().await.unwrap(),
2200            ConnectionEvent::ReconnectAttempt {
2201                attempt: 1,
2202                delay: Duration::ZERO,
2203            }
2204        );
2205        wait_for_calls(&calls, 2).await;
2206        assert!(client.is_reconnecting());
2207
2208        command.abort();
2209        assert!(command.await.unwrap_err().is_cancelled());
2210        drop(client);
2211        assert_eq!(
2212            stream.recv().await.unwrap(),
2213            ConnectionEvent::Disconnected {
2214                reason: ConnectionDisconnectReason::Shutdown,
2215            },
2216            "shutdown remains observable after the outage disconnect"
2217        );
2218        assert!(
2219            tokio::time::timeout(Duration::from_millis(10), stream.recv())
2220                .await
2221                .is_err(),
2222            "terminal shutdown must be published exactly once"
2223        );
2224    }
2225
2226    #[tokio::test]
2227    async fn shutdown_is_terminal_when_factory_and_queue_close_are_ready_together() {
2228        let (initial, initial_server) = connection_pair().await;
2229        let initial_seen = close_on_first_request(initial_server);
2230        let calls = Arc::new(AtomicUsize::new(0));
2231        let ready = Arc::new(AtomicBool::new(false));
2232        let factory = ReadyOnCloseFactory {
2233            calls: Arc::clone(&calls),
2234            initial: StdMutex::new(Some(initial)),
2235            ready: Arc::clone(&ready),
2236        };
2237        let events = ConnectionEventBus::new(8);
2238        let mut stream = events.subscribe();
2239        let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2240            factory,
2241            config(0, Duration::ZERO, None),
2242            OfflineQueueConfig::new(1),
2243            events.clone(),
2244        )
2245        .await
2246        .unwrap();
2247        assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2248
2249        let command_client = client.clone();
2250        let command = tokio::spawn(async move {
2251            command_client
2252                .execute(EchoCommand::new("simultaneous-close", true))
2253                .await
2254        });
2255        initial_seen.await.unwrap();
2256        assert!(matches!(
2257            stream.recv().await.unwrap(),
2258            ConnectionEvent::Disconnected {
2259                reason: ConnectionDisconnectReason::ConnectionError { .. }
2260            }
2261        ));
2262        assert_eq!(
2263            stream.recv().await.unwrap(),
2264            ConnectionEvent::ReconnectAttempt {
2265                attempt: 1,
2266                delay: Duration::ZERO,
2267            }
2268        );
2269        wait_for_calls(&calls, 2).await;
2270
2271        // Leave one handle in the queued command, make its factory branch
2272        // ready without waking it, then cancel that final handle. Queue close
2273        // wakes a select where both connect and close are ready.
2274        drop(client);
2275        ready.store(true, Ordering::Release);
2276        command.abort();
2277        assert!(command.await.unwrap_err().is_cancelled());
2278        assert_eq!(
2279            stream.recv().await.unwrap(),
2280            ConnectionEvent::Disconnected {
2281                reason: ConnectionDisconnectReason::Shutdown,
2282            }
2283        );
2284
2285        for _ in 0..4 {
2286            tokio::task::yield_now().await;
2287        }
2288        assert!(
2289            tokio::time::timeout(Duration::from_millis(10), stream.recv())
2290                .await
2291                .is_err(),
2292            "no reconnect event may be published after terminal shutdown"
2293        );
2294    }
2295
2296    // -- defect 3: single-flight gate --
2297
2298    #[tokio::test]
2299    async fn gate_skips_when_another_clone_already_reconnected() {
2300        let gate = ReconnectGate::new();
2301        // Two clones both observe generation 0 before either reconnects.
2302        let seen_a = gate.generation();
2303        let seen_b = gate.generation();
2304
2305        // Clone A reconnects: it sees no change, proceeds, and marks success.
2306        {
2307            let _g = gate.enter().await;
2308            assert_eq!(gate.generation(), seen_a);
2309            gate.mark_reconnected();
2310        }
2311
2312        // Clone B now takes the gate: the generation advanced, so it skips.
2313        {
2314            let _g = gate.enter().await;
2315            assert_ne!(gate.generation(), seen_b);
2316        }
2317    }
2318
2319    #[tokio::test]
2320    async fn gate_serializes_concurrent_holders() {
2321        let gate = Arc::new(ReconnectGate::new());
2322        let held = Arc::new(AtomicUsize::new(0));
2323        let max_seen = Arc::new(AtomicUsize::new(0));
2324
2325        let mut handles = Vec::new();
2326        for _ in 0..8 {
2327            let gate = gate.clone();
2328            let held = held.clone();
2329            let max_seen = max_seen.clone();
2330            handles.push(tokio::spawn(async move {
2331                let _g = gate.enter().await;
2332                let now = held.fetch_add(1, Ordering::SeqCst) + 1;
2333                max_seen.fetch_max(now, Ordering::SeqCst);
2334                tokio::task::yield_now().await;
2335                held.fetch_sub(1, Ordering::SeqCst);
2336            }));
2337        }
2338        for h in handles {
2339            h.await.unwrap();
2340        }
2341        // The gate is exclusive: never more than one holder at a time.
2342        assert_eq!(max_seen.load(Ordering::SeqCst), 1);
2343    }
2344
2345    fn wait_decision(decision: QueueDecision) -> (QueuePermit, Option<u64>) {
2346        match decision {
2347            QueueDecision::Wait { permit, campaign } => (permit, campaign),
2348            QueueDecision::Execute => panic!("expected queued decision, got execute"),
2349            QueueDecision::Reject { error, .. } => {
2350                panic!("expected queued decision, got {error:?}")
2351            }
2352        }
2353    }
2354
2355    #[test]
2356    fn offline_queue_config_is_explicit_and_defaults_to_a_bound() {
2357        assert_eq!(OfflineQueueConfig::new(7).capacity(), 7);
2358        assert_eq!(OfflineQueueConfig::new(7).max_replay_attempts(), 3);
2359        assert_eq!(
2360            OfflineQueueConfig::new(7)
2361                .with_max_replay_attempts(5)
2362                .max_replay_attempts(),
2363            5
2364        );
2365        assert_eq!(OfflineQueueConfig::default().capacity(), 1024);
2366
2367        let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(0));
2368        assert!(matches!(
2369            queue.after_connection_error(true),
2370            QueueDecision::Reject {
2371                error: QueueRejection::Full,
2372                campaign: Some(0),
2373            }
2374        ));
2375        assert!(queue.is_reconnecting());
2376    }
2377
2378    #[tokio::test]
2379    async fn offline_queue_is_bounded_ticket_ordered_and_cancellation_safe() {
2380        let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(3));
2381        let (first, campaign) = wait_decision(queue.after_connection_error(true));
2382        assert_eq!(campaign, Some(0));
2383        let (second, campaign) = wait_decision(queue.before_execute(true));
2384        assert_eq!(campaign, None);
2385        let (third, campaign) = wait_decision(queue.before_execute(true));
2386        assert_eq!(campaign, None);
2387        assert_eq!(queue.depth(), 3);
2388
2389        assert!(matches!(
2390            queue.before_execute(true),
2391            QueueDecision::Reject {
2392                error: QueueRejection::Full,
2393                campaign: None,
2394            }
2395        ));
2396        assert!(matches!(
2397            queue.before_execute(false),
2398            QueueDecision::Reject {
2399                error: QueueRejection::ConnectionClosed,
2400                campaign: None,
2401            }
2402        ));
2403
2404        queue.inner.finish_reconnect(0, None);
2405        first.wait_for_turn().await.unwrap();
2406        assert!(
2407            tokio::time::timeout(Duration::from_millis(10), second.wait_for_turn())
2408                .await
2409                .is_err(),
2410            "the second ticket must not overtake the first"
2411        );
2412
2413        // Canceling the head removes it synchronously and wakes its successor.
2414        drop(first);
2415        second.wait_for_turn().await.unwrap();
2416        drop(second);
2417        third.wait_for_turn().await.unwrap();
2418        drop(third);
2419        assert_eq!(queue.depth(), 0);
2420        assert!(matches!(queue.state().phase, OfflinePhase::Connected));
2421    }
2422
2423    #[tokio::test]
2424    async fn failed_campaign_rejects_every_admitted_ticket_deterministically() {
2425        let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(2));
2426        let (first, campaign) = wait_decision(queue.after_connection_error(true));
2427        let campaign = campaign.unwrap();
2428        let (second, _) = wait_decision(queue.before_execute(true));
2429        let cause = Arc::new(RedisError::ConnectTimeout);
2430        queue.inner.finish_reconnect(
2431            campaign,
2432            Some(ReconnectFailure {
2433                attempts: 3,
2434                last_error: Arc::clone(&cause),
2435            }),
2436        );
2437
2438        let mut waiter_causes = Vec::new();
2439        for permit in [&first, &second] {
2440            let error = permit.wait_for_turn().await.unwrap_err();
2441            let RedisError::ReconnectFailed {
2442                attempts: 3,
2443                last_error,
2444            } = error
2445            else {
2446                panic!("expected structured reconnect exhaustion")
2447            };
2448            assert!(matches!(&*last_error, RedisError::ConnectTimeout));
2449            assert!(Arc::ptr_eq(&last_error, &cause));
2450            waiter_causes.push(last_error);
2451        }
2452        assert!(Arc::ptr_eq(&waiter_causes[0], &waiter_causes[1]));
2453
2454        let QueueDecision::Reject {
2455            error:
2456                QueueRejection::ReconnectFailed {
2457                    attempts: 3,
2458                    last_error,
2459                },
2460            campaign: None,
2461        } = queue.before_execute(true)
2462        else {
2463            panic!("new work should observe the same reconnect exhaustion")
2464        };
2465        assert!(matches!(&*last_error, RedisError::ConnectTimeout));
2466        assert!(Arc::ptr_eq(&last_error, &cause));
2467
2468        assert!(matches!(
2469            queue.before_execute(false),
2470            QueueDecision::Reject {
2471                error: QueueRejection::ConnectionClosed,
2472                campaign: None,
2473            }
2474        ));
2475
2476        drop(first);
2477        drop(second);
2478        let (replacement, next_campaign) = wait_decision(queue.before_execute(true));
2479        assert_eq!(next_campaign, Some(1));
2480        drop(replacement);
2481    }
2482
2483    #[tokio::test]
2484    async fn closing_queue_wakes_waiters_with_connection_closed() {
2485        let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(1));
2486        let (permit, _) = wait_decision(queue.after_connection_error(true));
2487        queue.inner.close();
2488
2489        assert!(matches!(
2490            permit.wait_for_turn().await,
2491            Err(RedisError::ConnectionClosed)
2492        ));
2493        assert!(matches!(
2494            queue.before_execute(true),
2495            QueueDecision::Reject {
2496                error: QueueRejection::ConnectionClosed,
2497                campaign: None,
2498            }
2499        ));
2500    }
2501
2502    #[tokio::test]
2503    async fn dropping_last_queue_owner_cancels_hanging_reconnect() {
2504        let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(1));
2505        let mut closed = queue.inner.closed_tx.subscribe();
2506        let calls = Arc::new(AtomicUsize::new(0));
2507        let factory = HangingFactory {
2508            calls: Arc::clone(&calls),
2509        };
2510        let config = config(0, Duration::ZERO, None);
2511
2512        let task = tokio::spawn(async move {
2513            reconnect_campaign_until_closed(&factory, &config, &mut closed, None).await
2514        });
2515        while calls.load(Ordering::SeqCst) == 0 {
2516            tokio::task::yield_now().await;
2517        }
2518        drop(queue);
2519
2520        assert!(matches!(
2521            tokio::time::timeout(Duration::from_secs(1), task)
2522                .await
2523                .expect("queue closure should cancel a hanging connect")
2524                .unwrap(),
2525            CancellableReconnect::Closed
2526        ));
2527    }
2528
2529    #[tokio::test]
2530    async fn idempotent_commands_replay_in_ticket_order_with_one_campaign() {
2531        let (initial, initial_server) = connection_pair().await;
2532        let initial_seen = close_on_first_request(initial_server);
2533        let (replacement, replacement_server) = connection_pair().await;
2534        let replacement_task = echo_server(replacement_server, 3);
2535
2536        let calls = Arc::new(AtomicUsize::new(0));
2537        let release = Arc::new(Semaphore::new(0));
2538        let factory = ControlledFactory {
2539            calls: Arc::clone(&calls),
2540            initial: StdMutex::new(Some(initial)),
2541            replacement: Arc::new(StdMutex::new(Some(replacement))),
2542            release_reconnect: Arc::clone(&release),
2543        };
2544        let client = ResilientRedisClient::with_config_and_offline_queue(
2545            factory,
2546            config(0, Duration::ZERO, None),
2547            OfflineQueueConfig::new(3),
2548        )
2549        .await
2550        .unwrap();
2551
2552        let first_client = client.clone();
2553        let first =
2554            tokio::spawn(
2555                async move { first_client.execute(EchoCommand::new("first", true)).await },
2556            );
2557        initial_seen.await.unwrap();
2558        wait_for_depth(&client, 1).await;
2559        assert!(client.is_reconnecting());
2560
2561        let second_client = client.clone();
2562        let second = tokio::spawn(async move {
2563            second_client
2564                .execute(EchoCommand::new("second", true))
2565                .await
2566        });
2567        wait_for_depth(&client, 2).await;
2568        let third_client = client.clone();
2569        let third =
2570            tokio::spawn(
2571                async move { third_client.execute(EchoCommand::new("third", true)).await },
2572            );
2573        wait_for_depth(&client, 3).await;
2574
2575        assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
2576        release.add_permits(1);
2577
2578        assert_eq!(first.await.unwrap().unwrap(), Bytes::from_static(b"first"));
2579        assert_eq!(
2580            second.await.unwrap().unwrap(),
2581            Bytes::from_static(b"second")
2582        );
2583        assert_eq!(third.await.unwrap().unwrap(), Bytes::from_static(b"third"));
2584        assert_eq!(
2585            replacement_task.await.unwrap(),
2586            vec![
2587                Bytes::from_static(b"first"),
2588                Bytes::from_static(b"second"),
2589                Bytes::from_static(b"third"),
2590            ]
2591        );
2592        assert_eq!(client.offline_queue_depth(), 0);
2593        assert!(!client.is_reconnecting());
2594    }
2595
2596    #[tokio::test]
2597    async fn offline_queue_campaign_publishes_ordered_lifecycle_events() {
2598        use crate::reconnect::ConnectionDisconnectReason;
2599
2600        let (initial, initial_server) = connection_pair().await;
2601        let initial_seen = close_on_first_request(initial_server);
2602        let (replacement, replacement_server) = connection_pair().await;
2603        let replacement_task = echo_server(replacement_server, 1);
2604        let release = Arc::new(Semaphore::new(0));
2605        let factory = ControlledFactory {
2606            calls: Arc::new(AtomicUsize::new(0)),
2607            initial: StdMutex::new(Some(initial)),
2608            replacement: Arc::new(StdMutex::new(Some(replacement))),
2609            release_reconnect: Arc::clone(&release),
2610        };
2611        let events = ConnectionEventBus::new(8);
2612        let mut event_stream = events.subscribe();
2613        let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2614            factory,
2615            config(0, Duration::ZERO, None),
2616            OfflineQueueConfig::new(1),
2617            events,
2618        )
2619        .await
2620        .unwrap();
2621        assert_eq!(
2622            event_stream.recv().await.unwrap(),
2623            ConnectionEvent::Connected
2624        );
2625
2626        let command_client = client.clone();
2627        let command = tokio::spawn(async move {
2628            command_client
2629                .execute(EchoCommand::new("eventful", true))
2630                .await
2631        });
2632        initial_seen.await.unwrap();
2633        assert!(matches!(
2634            event_stream.recv().await.unwrap(),
2635            ConnectionEvent::Disconnected {
2636                reason: ConnectionDisconnectReason::ConnectionError { .. }
2637            }
2638        ));
2639        assert_eq!(
2640            event_stream.recv().await.unwrap(),
2641            ConnectionEvent::ReconnectAttempt {
2642                attempt: 1,
2643                delay: Duration::ZERO,
2644            }
2645        );
2646
2647        release.add_permits(1);
2648        assert_eq!(
2649            command.await.unwrap().unwrap(),
2650            Bytes::from_static(b"eventful")
2651        );
2652        assert!(matches!(
2653            event_stream.recv().await.unwrap(),
2654            ConnectionEvent::Reconnected { attempts: 1, .. }
2655        ));
2656        assert_eq!(
2657            replacement_task.await.unwrap(),
2658            vec![Bytes::from_static(b"eventful")]
2659        );
2660    }
2661
2662    #[tokio::test]
2663    async fn default_client_reconnects_but_does_not_replay_idempotent_command() {
2664        let (initial, initial_server) = connection_pair().await;
2665        let initial_seen = close_on_first_request(initial_server);
2666        let (replacement, mut replacement_server) = connection_pair().await;
2667
2668        let calls = Arc::new(AtomicUsize::new(0));
2669        let release = Arc::new(Semaphore::new(0));
2670        let factory = ControlledFactory {
2671            calls: Arc::clone(&calls),
2672            initial: StdMutex::new(Some(initial)),
2673            replacement: Arc::new(StdMutex::new(Some(replacement))),
2674            release_reconnect: Arc::clone(&release),
2675        };
2676        let client = ResilientRedisClient::with_config(factory, config(0, Duration::ZERO, None))
2677            .await
2678            .unwrap();
2679
2680        let command_client = client.clone();
2681        let command = tokio::spawn(async move {
2682            command_client
2683                .execute(EchoCommand::new("not-replayed", true))
2684                .await
2685        });
2686        initial_seen.await.unwrap();
2687        tokio::time::timeout(Duration::from_secs(1), async {
2688            while calls.load(Ordering::SeqCst) < 2 {
2689                tokio::task::yield_now().await;
2690            }
2691        })
2692        .await
2693        .expect("default reconnect campaign did not start");
2694        assert!(client.is_reconnecting());
2695        release.add_permits(1);
2696
2697        assert!(command.await.unwrap().is_err());
2698        let mut bytes = [0; 128];
2699        assert!(
2700            tokio::time::timeout(
2701                Duration::from_millis(25),
2702                replacement_server.read(&mut bytes)
2703            )
2704            .await
2705            .is_err(),
2706            "default construction must not replay the failed command"
2707        );
2708        assert_eq!(client.offline_queue_depth(), 0);
2709        assert!(!client.is_reconnecting());
2710    }
2711
2712    #[tokio::test]
2713    async fn non_idempotent_commands_fail_instead_of_waiting_or_replaying() {
2714        let (initial, initial_server) = connection_pair().await;
2715        let initial_seen = close_on_first_request(initial_server);
2716        let (replacement, replacement_server) = connection_pair().await;
2717        let replacement_task = echo_server(replacement_server, 1);
2718
2719        let calls = Arc::new(AtomicUsize::new(0));
2720        let release = Arc::new(Semaphore::new(0));
2721        let factory = ControlledFactory {
2722            calls: Arc::clone(&calls),
2723            initial: StdMutex::new(Some(initial)),
2724            replacement: Arc::new(StdMutex::new(Some(replacement))),
2725            release_reconnect: Arc::clone(&release),
2726        };
2727        let client = ResilientRedisClient::with_config_and_offline_queue(
2728            factory,
2729            config(0, Duration::ZERO, None),
2730            OfflineQueueConfig::new(2),
2731        )
2732        .await
2733        .unwrap();
2734
2735        let discovering_client = client.clone();
2736        let discovering = tokio::spawn(async move {
2737            discovering_client
2738                .execute(EchoCommand::new("unsafe-first", false))
2739                .await
2740        });
2741        initial_seen.await.unwrap();
2742        assert!(discovering.await.unwrap().is_err());
2743
2744        tokio::time::timeout(Duration::from_secs(1), async {
2745            while !client.is_reconnecting() {
2746                tokio::task::yield_now().await;
2747            }
2748        })
2749        .await
2750        .expect("reconnect campaign did not start");
2751
2752        let error = client
2753            .execute(EchoCommand::new("unsafe-while-offline", false))
2754            .await
2755            .unwrap_err();
2756        assert!(matches!(error, RedisError::ConnectionClosed));
2757        let mut raw_client = client.clone();
2758        let raw_error = raw_client
2759            .call(array(vec![bulk("ECHO"), bulk("raw-while-offline")]))
2760            .await
2761            .unwrap_err();
2762        assert!(matches!(raw_error, RedisError::ConnectionClosed));
2763        assert_eq!(client.offline_queue_depth(), 0);
2764
2765        let safe_client = client.clone();
2766        let safe =
2767            tokio::spawn(async move { safe_client.execute(EchoCommand::new("safe", true)).await });
2768        wait_for_depth(&client, 1).await;
2769        release.add_permits(1);
2770        assert_eq!(safe.await.unwrap().unwrap(), Bytes::from_static(b"safe"));
2771        assert_eq!(
2772            replacement_task.await.unwrap(),
2773            vec![Bytes::from_static(b"safe")],
2774            "neither non-idempotent command may be replayed"
2775        );
2776        assert_eq!(calls.load(Ordering::SeqCst), 2);
2777    }
2778
2779    #[tokio::test]
2780    async fn already_expired_offline_command_never_reaches_wire() {
2781        let (connection, mut server) = connection_pair().await;
2782        let initial = Arc::new(StdMutex::new(Some(connection)));
2783        let factory = {
2784            let initial = Arc::clone(&initial);
2785            move || {
2786                let connection = initial
2787                    .lock()
2788                    .expect("initial connection mutex poisoned")
2789                    .take();
2790                async move { connection.ok_or(RedisError::ConnectionClosed) }
2791            }
2792        };
2793        let client = ResilientRedisClient::with_config_and_offline_queue(
2794            factory,
2795            config(0, Duration::ZERO, None),
2796            OfflineQueueConfig::new(1),
2797        )
2798        .await
2799        .unwrap();
2800
2801        let result = client
2802            .execute(WithDeadline::new(
2803                EchoCommand::new("expired", true),
2804                tokio::time::Instant::now() - Duration::from_millis(1),
2805            ))
2806            .await;
2807
2808        assert!(matches!(result, Err(RedisError::CommandTimeout)));
2809        assert_eq!(client.offline_queue_depth(), 0);
2810        assert!(!client.is_reconnecting());
2811        let mut bytes = [0; 128];
2812        assert!(
2813            tokio::time::timeout(Duration::from_millis(25), server.read(&mut bytes))
2814                .await
2815                .is_err(),
2816            "an already-expired command reached the socket"
2817        );
2818    }
2819
2820    #[tokio::test]
2821    async fn queued_deadline_expiry_releases_ticket_without_replay() {
2822        let (initial, initial_server) = connection_pair().await;
2823        let initial_seen = close_on_first_request(initial_server);
2824        let (replacement, replacement_server) = connection_pair().await;
2825        let replacement_task = echo_server(replacement_server, 1);
2826
2827        let calls = Arc::new(AtomicUsize::new(0));
2828        let release = Arc::new(Semaphore::new(0));
2829        let factory = ControlledFactory {
2830            calls: Arc::clone(&calls),
2831            initial: StdMutex::new(Some(initial)),
2832            replacement: Arc::new(StdMutex::new(Some(replacement))),
2833            release_reconnect: Arc::clone(&release),
2834        };
2835        let client = ResilientRedisClient::with_config_and_offline_queue(
2836            factory,
2837            config(0, Duration::ZERO, None),
2838            OfflineQueueConfig::new(2),
2839        )
2840        .await
2841        .unwrap();
2842
2843        let expiring_client = client.clone();
2844        let expiring = tokio::spawn(async move {
2845            expiring_client
2846                .execute(WithDeadline::after(
2847                    EchoCommand::new("expired", true),
2848                    Duration::from_millis(250),
2849                ))
2850                .await
2851        });
2852        initial_seen.await.unwrap();
2853        wait_for_depth(&client, 1).await;
2854
2855        assert!(matches!(
2856            expiring.await.unwrap(),
2857            Err(RedisError::CommandTimeout)
2858        ));
2859        wait_for_depth(&client, 0).await;
2860
2861        // Finish the shared reconnect after the expired ticket has left the
2862        // queue. Only the later command may reach the replacement socket.
2863        wait_for_calls(&calls, 2).await;
2864        release.add_permits(1);
2865        assert_eq!(
2866            client
2867                .execute(EchoCommand::new("later", true))
2868                .await
2869                .unwrap(),
2870            Bytes::from_static(b"later")
2871        );
2872        assert_eq!(
2873            replacement_task.await.unwrap(),
2874            vec![Bytes::from_static(b"later")]
2875        );
2876        assert_eq!(client.offline_queue_depth(), 0);
2877    }
2878
2879    #[tokio::test]
2880    async fn in_wire_deadline_timeout_quarantines_socket_before_successor() {
2881        let (initial, initial_server) = connection_pair().await;
2882        let (initial_seen, held_server) = hold_after_first_request(initial_server);
2883        let (replacement, replacement_server) = connection_pair().await;
2884        let replacement_task = echo_server(replacement_server, 1);
2885
2886        let calls = Arc::new(AtomicUsize::new(0));
2887        let release = Arc::new(Semaphore::new(0));
2888        let factory = ControlledFactory {
2889            calls: Arc::clone(&calls),
2890            initial: StdMutex::new(Some(initial)),
2891            replacement: Arc::new(StdMutex::new(Some(replacement))),
2892            release_reconnect: Arc::clone(&release),
2893        };
2894        let client = ResilientRedisClient::with_config_and_offline_queue(
2895            factory,
2896            config(0, Duration::ZERO, None),
2897            OfflineQueueConfig::new(2),
2898        )
2899        .await
2900        .unwrap();
2901
2902        let timed_client = client.clone();
2903        let timed = tokio::spawn(async move {
2904            timed_client
2905                .execute(WithDeadline::after(
2906                    EchoCommand::new("timed-out", true),
2907                    Duration::from_millis(250),
2908                ))
2909                .await
2910        });
2911        initial_seen.await.unwrap();
2912        assert!(matches!(
2913            timed.await.unwrap(),
2914            Err(RedisError::CommandTimeout)
2915        ));
2916
2917        // ProtocolAlignmentGuard starts replacement synchronously when the
2918        // deadline drops an exchange that was already on the wire.
2919        wait_for_calls(&calls, 2).await;
2920        assert!(client.is_reconnecting());
2921        let later_client = client.clone();
2922        let later =
2923            tokio::spawn(
2924                async move { later_client.execute(EchoCommand::new("later", true)).await },
2925            );
2926        wait_for_depth(&client, 1).await;
2927        release.add_permits(1);
2928
2929        assert_eq!(later.await.unwrap().unwrap(), Bytes::from_static(b"later"));
2930        assert_eq!(
2931            replacement_task.await.unwrap(),
2932            vec![Bytes::from_static(b"later")]
2933        );
2934        assert_eq!(calls.load(Ordering::SeqCst), 2);
2935        assert_eq!(client.offline_queue_depth(), 0);
2936        assert!(!client.is_reconnecting());
2937        held_server.abort();
2938    }
2939
2940    #[tokio::test]
2941    async fn canceling_head_command_does_not_cancel_shared_reconnect() {
2942        let (initial, initial_server) = connection_pair().await;
2943        let initial_seen = close_on_first_request(initial_server);
2944        let (replacement, replacement_server) = connection_pair().await;
2945        let replacement_task = echo_server(replacement_server, 1);
2946
2947        let calls = Arc::new(AtomicUsize::new(0));
2948        let release = Arc::new(Semaphore::new(0));
2949        let factory = ControlledFactory {
2950            calls: Arc::clone(&calls),
2951            initial: StdMutex::new(Some(initial)),
2952            replacement: Arc::new(StdMutex::new(Some(replacement))),
2953            release_reconnect: Arc::clone(&release),
2954        };
2955        let client = ResilientRedisClient::with_config_and_offline_queue(
2956            factory,
2957            config(0, Duration::ZERO, None),
2958            OfflineQueueConfig::new(2),
2959        )
2960        .await
2961        .unwrap();
2962
2963        let canceled_client = client.clone();
2964        let canceled = tokio::spawn(async move {
2965            canceled_client
2966                .execute(EchoCommand::new("canceled", true))
2967                .await
2968        });
2969        initial_seen.await.unwrap();
2970        wait_for_depth(&client, 1).await;
2971
2972        let surviving_client = client.clone();
2973        let surviving = tokio::spawn(async move {
2974            surviving_client
2975                .execute(EchoCommand::new("survives", true))
2976                .await
2977        });
2978        wait_for_depth(&client, 2).await;
2979        assert!(matches!(
2980            client.execute(EchoCommand::new("overflow", true)).await,
2981            Err(RedisError::QueueFull)
2982        ));
2983        canceled.abort();
2984        assert!(canceled.await.unwrap_err().is_cancelled());
2985        wait_for_depth(&client, 1).await;
2986
2987        release.add_permits(1);
2988        assert_eq!(
2989            surviving.await.unwrap().unwrap(),
2990            Bytes::from_static(b"survives")
2991        );
2992        assert_eq!(
2993            replacement_task.await.unwrap(),
2994            vec![Bytes::from_static(b"survives")]
2995        );
2996        assert_eq!(calls.load(Ordering::SeqCst), 2);
2997    }
2998
2999    #[tokio::test]
3000    async fn canceling_initial_wire_attempt_replaces_socket_before_later_command() {
3001        let (initial, initial_server) = connection_pair().await;
3002        let (initial_seen, held_server) = hold_after_first_request(initial_server);
3003        let (replacement, replacement_server) = connection_pair().await;
3004        let replacement_task = echo_server(replacement_server, 1);
3005
3006        let calls = Arc::new(AtomicUsize::new(0));
3007        let release = Arc::new(Semaphore::new(0));
3008        let factory = ControlledFactory {
3009            calls: Arc::clone(&calls),
3010            initial: StdMutex::new(Some(initial)),
3011            replacement: Arc::new(StdMutex::new(Some(replacement))),
3012            release_reconnect: Arc::clone(&release),
3013        };
3014        let client = ResilientRedisClient::with_config_and_offline_queue(
3015            factory,
3016            config(0, Duration::ZERO, None),
3017            OfflineQueueConfig::new(2),
3018        )
3019        .await
3020        .unwrap();
3021
3022        let canceled_client = client.clone();
3023        let canceled = tokio::spawn(async move {
3024            canceled_client
3025                .execute(EchoCommand::new("abandoned", true))
3026                .await
3027        });
3028        initial_seen.await.unwrap();
3029        canceled.abort();
3030        assert!(canceled.await.unwrap_err().is_cancelled());
3031
3032        // Dropping the in-wire future synchronously starts one replacement
3033        // campaign. A later command waits rather than reading the abandoned
3034        // ECHO response from the original socket.
3035        wait_for_calls(&calls, 2).await;
3036        assert!(client.is_reconnecting());
3037        let later_client = client.clone();
3038        let later =
3039            tokio::spawn(
3040                async move { later_client.execute(EchoCommand::new("later", true)).await },
3041            );
3042        wait_for_depth(&client, 1).await;
3043        release.add_permits(1);
3044
3045        assert_eq!(later.await.unwrap().unwrap(), Bytes::from_static(b"later"));
3046        assert_eq!(
3047            replacement_task.await.unwrap(),
3048            vec![Bytes::from_static(b"later")]
3049        );
3050        assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
3051        assert_eq!(client.offline_queue_depth(), 0);
3052        assert!(!client.is_reconnecting());
3053        held_server.abort();
3054    }
3055
3056    #[tokio::test]
3057    async fn canceling_raw_wire_attempt_replaces_socket_before_waiting_successor() {
3058        let (initial, initial_server) = connection_pair().await;
3059        let (initial_seen, send_late_response, late_response_task) =
3060            delayed_response_after_first_request(
3061                initial_server,
3062                Bytes::from_static(b"abandoned-raw"),
3063            );
3064        let (replacement, replacement_server) = connection_pair().await;
3065        let replacement_task = echo_server(replacement_server, 1);
3066
3067        let calls = Arc::new(AtomicUsize::new(0));
3068        let release = Arc::new(Semaphore::new(0));
3069        let factory = ControlledFactory {
3070            calls: Arc::clone(&calls),
3071            initial: StdMutex::new(Some(initial)),
3072            replacement: Arc::new(StdMutex::new(Some(replacement))),
3073            release_reconnect: Arc::clone(&release),
3074        };
3075        let client = ResilientRedisClient::with_config_and_offline_queue(
3076            factory,
3077            config(0, Duration::ZERO, None),
3078            OfflineQueueConfig::new(2),
3079        )
3080        .await
3081        .unwrap();
3082
3083        let mut raw_client = client.clone();
3084        let canceled = tokio::spawn(async move {
3085            raw_client
3086                .call(array(vec![bulk("ECHO"), bulk("abandoned-raw")]))
3087                .await
3088        });
3089        initial_seen.await.unwrap();
3090
3091        // Put the successor behind the raw call's connection mutex before
3092        // aborting it. Guard drop must mark the connection offline before that
3093        // mutex is released, forcing the successor into the queue.
3094        let later_client = client.clone();
3095        let later = tokio::spawn(async move {
3096            later_client
3097                .execute(EchoCommand::new("after-raw", true))
3098                .await
3099        });
3100        tokio::task::yield_now().await;
3101        assert!(
3102            !later.is_finished(),
3103            "successor should wait behind raw request"
3104        );
3105
3106        canceled.abort();
3107        assert!(canceled.await.unwrap_err().is_cancelled());
3108        wait_for_calls(&calls, 2).await;
3109        wait_for_depth(&client, 1).await;
3110
3111        // Deliver the abandoned raw response while the replacement factory is
3112        // still gated. It remains buffered on the quarantined socket and must
3113        // never satisfy the already-waiting successor.
3114        send_late_response.send(()).unwrap();
3115        late_response_task.await.unwrap();
3116        assert!(!later.is_finished());
3117        release.add_permits(1);
3118
3119        assert_eq!(
3120            later.await.unwrap().unwrap(),
3121            Bytes::from_static(b"after-raw")
3122        );
3123        assert_eq!(
3124            replacement_task.await.unwrap(),
3125            vec![Bytes::from_static(b"after-raw")]
3126        );
3127        assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
3128        assert_eq!(client.offline_queue_depth(), 0);
3129        assert!(!client.is_reconnecting());
3130    }
3131
3132    #[tokio::test]
3133    async fn canceling_queued_wire_replay_replaces_socket_before_next_ticket() {
3134        let (initial, initial_server) = connection_pair().await;
3135        let initial_seen = close_on_first_request(initial_server);
3136        let (first_replacement, first_replacement_server) = connection_pair().await;
3137        let (replay_seen, held_replay_server) = hold_after_first_request(first_replacement_server);
3138        let (fresh_replacement, fresh_replacement_server) = connection_pair().await;
3139        let fresh_server_task = echo_server(fresh_replacement_server, 1);
3140
3141        let calls = Arc::new(AtomicUsize::new(0));
3142        let release = Arc::new(Semaphore::new(0));
3143        let factory = SequencedFactory {
3144            calls: Arc::clone(&calls),
3145            initial: StdMutex::new(Some(initial)),
3146            replacements: Arc::new(StdMutex::new(VecDeque::from([
3147                first_replacement,
3148                fresh_replacement,
3149            ]))),
3150            release_reconnect: Arc::clone(&release),
3151        };
3152        let client = ResilientRedisClient::with_config_and_offline_queue(
3153            factory,
3154            config(0, Duration::ZERO, None),
3155            OfflineQueueConfig::new(2),
3156        )
3157        .await
3158        .unwrap();
3159
3160        let canceled_client = client.clone();
3161        let canceled = tokio::spawn(async move {
3162            canceled_client
3163                .execute(EchoCommand::new("head", true))
3164                .await
3165        });
3166        initial_seen.await.unwrap();
3167        wait_for_depth(&client, 1).await;
3168        wait_for_calls(&calls, 2).await;
3169        release.add_permits(1);
3170        replay_seen.await.unwrap();
3171
3172        canceled.abort();
3173        assert!(canceled.await.unwrap_err().is_cancelled());
3174        wait_for_calls(&calls, 3).await;
3175        assert!(client.is_reconnecting());
3176
3177        let later_client = client.clone();
3178        let later = tokio::spawn(async move {
3179            later_client
3180                .execute(EchoCommand::new("next-ticket", true))
3181                .await
3182        });
3183        wait_for_depth(&client, 1).await;
3184        release.add_permits(1);
3185
3186        assert_eq!(
3187            later.await.unwrap().unwrap(),
3188            Bytes::from_static(b"next-ticket")
3189        );
3190        assert_eq!(
3191            fresh_server_task.await.unwrap(),
3192            vec![Bytes::from_static(b"next-ticket")]
3193        );
3194        assert_eq!(calls.load(Ordering::SeqCst), 3, "two reconnect campaigns");
3195        assert_eq!(client.offline_queue_depth(), 0);
3196        assert!(!client.is_reconnecting());
3197        held_replay_server.abort();
3198    }
3199
3200    #[tokio::test]
3201    async fn replay_attempt_limit_bounds_successful_connect_disconnect_churn() {
3202        let (initial, initial_server) = connection_pair().await;
3203        let initial_seen = close_on_first_request(initial_server);
3204        let (replacement_one, server_one) = connection_pair().await;
3205        let replay_one_seen = close_on_first_request(server_one);
3206        let (replacement_two, server_two) = connection_pair().await;
3207        let replay_two_seen = close_on_first_request(server_two);
3208        let (repair, repair_server) = connection_pair().await;
3209        let repair_task = echo_server(repair_server, 1);
3210
3211        let calls = Arc::new(AtomicUsize::new(0));
3212        let release = Arc::new(Semaphore::new(0));
3213        let factory = SequencedFactory {
3214            calls: Arc::clone(&calls),
3215            initial: StdMutex::new(Some(initial)),
3216            replacements: Arc::new(StdMutex::new(VecDeque::from([
3217                replacement_one,
3218                replacement_two,
3219                repair,
3220            ]))),
3221            release_reconnect: Arc::clone(&release),
3222        };
3223        let client = ResilientRedisClient::with_config_and_offline_queue(
3224            factory,
3225            config(0, Duration::ZERO, None),
3226            OfflineQueueConfig::new(2).with_max_replay_attempts(2),
3227        )
3228        .await
3229        .unwrap();
3230
3231        let head_client = client.clone();
3232        let head = tokio::spawn(async move {
3233            head_client
3234                .execute(EchoCommand::new("bounded-head", true))
3235                .await
3236        });
3237        initial_seen.await.unwrap();
3238        wait_for_calls(&calls, 2).await;
3239        release.add_permits(1);
3240        replay_one_seen.await.unwrap();
3241        wait_for_calls(&calls, 3).await;
3242        release.add_permits(1);
3243        replay_two_seen.await.unwrap();
3244
3245        let error = head.await.unwrap().unwrap_err();
3246        assert!(matches!(
3247            error,
3248            RedisError::ReconnectFailed { attempts: 2, .. }
3249        ));
3250        wait_for_calls(&calls, 4).await;
3251        assert_eq!(client.offline_queue_depth(), 0);
3252        assert!(client.is_reconnecting());
3253
3254        let later_client = client.clone();
3255        let later = tokio::spawn(async move {
3256            later_client
3257                .execute(EchoCommand::new("after-limit", true))
3258                .await
3259        });
3260        wait_for_depth(&client, 1).await;
3261        release.add_permits(1);
3262        assert_eq!(
3263            later.await.unwrap().unwrap(),
3264            Bytes::from_static(b"after-limit")
3265        );
3266        assert_eq!(
3267            repair_task.await.unwrap(),
3268            vec![Bytes::from_static(b"after-limit")]
3269        );
3270        assert_eq!(calls.load(Ordering::SeqCst), 4);
3271        assert_eq!(client.offline_queue_depth(), 0);
3272        assert!(!client.is_reconnecting());
3273    }
3274}