Skip to main content

redis_tower/
reconnect_layer.rs

1//! Tower Layer for automatic reconnection at the Frame level.
2//!
3//! Wraps a `FrameService` and reconnects when connection errors occur.
4//! Composes with other Frame-level middleware (caching, metrics).
5//!
6//! # Example
7//!
8//! ```no_run
9//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
10//! use redis_tower::cache_layer::{CacheConfig, CacheService};
11//! use redis_tower::command_adapter::CommandAdapter;
12//! use redis_tower::reconnect::{AddrConnectionFactory, ReconnectConfig};
13//! use redis_tower::reconnect_layer::ReconnectService;
14//!
15//! let factory = AddrConnectionFactory::new("127.0.0.1:6379");
16//!
17//! let svc = CommandAdapter::new(
18//!     CacheService::new(
19//!         ReconnectService::new(factory, ReconnectConfig::default()).await?,
20//!         CacheConfig::default(),
21//!     )
22//! );
23//! # let _ = svc;
24//! # Ok(())
25//! # }
26//! ```
27
28use std::future::Future;
29use std::pin::Pin;
30use std::sync::Arc;
31use std::sync::Mutex as StdMutex;
32use std::sync::atomic::{AtomicBool, Ordering};
33use std::task::{Context, Poll};
34use std::time::Instant;
35
36use redis_tower_core::{Frame, FrameService, RedisError};
37use tower_service::Service;
38
39use crate::reconnect::{
40    ConnectionEvent, ConnectionEventBus, ConnectionFactory, ReconnectConfig, connect_with_timeout,
41    publish_disconnect_before_shutdown, publish_shutdown_once,
42};
43
44type ReconnectFuture = Pin<Box<dyn Future<Output = Result<FrameService, RedisError>> + Send>>;
45
46enum State {
47    Ready,
48    WaitingToReconnect {
49        attempt: usize,
50        sleep: Pin<Box<tokio::time::Sleep>>,
51        started: Instant,
52    },
53    Reconnecting {
54        attempt: usize,
55        future: ReconnectFuture,
56        started: Instant,
57    },
58    Failed,
59}
60
61/// A `Service<Frame>` that automatically reconnects on connection errors.
62///
63/// Wraps a `FrameService` directly. When a connection error is detected,
64/// `poll_ready` drives the reconnection state machine with configurable
65/// exponential backoff.
66///
67/// # Factory Selection
68///
69/// The [`ConnectionFactory`] you provide determines what negotiation
70/// happens on each reconnect. Use [`UrlConnectionFactory`](crate::reconnect::UrlConnectionFactory)
71/// if your server requires AUTH or a specific database, so those are
72/// replayed on every new connection. See the [`reconnect`](crate::reconnect)
73/// module docs for the full factory comparison table.
74pub struct ReconnectService {
75    inner: FrameService,
76    factory: Arc<dyn ConnectionFactory>,
77    config: ReconnectConfig,
78    state: State,
79    needs_reconnect: Arc<AtomicBool>,
80    event_bus: Option<ConnectionEventBus>,
81    disconnect_reported: Option<Arc<AtomicBool>>,
82    shutdown_reported: Option<Arc<AtomicBool>>,
83    lifecycle_lock: Option<Arc<StdMutex<()>>>,
84}
85
86impl ReconnectService {
87    /// Create a new reconnecting service.
88    pub async fn new(
89        factory: impl ConnectionFactory,
90        config: ReconnectConfig,
91    ) -> Result<Self, RedisError> {
92        Self::new_inner(factory, config, None).await
93    }
94
95    /// Create a reconnecting frame service that publishes lifecycle events.
96    ///
97    /// Subscribe before calling this constructor to observe the initial
98    /// [`ConnectionEvent::Connected`] or [`ConnectionEvent::ConnectFailed`]
99    /// event.
100    pub async fn new_with_events(
101        factory: impl ConnectionFactory,
102        config: ReconnectConfig,
103        events: ConnectionEventBus,
104    ) -> Result<Self, RedisError> {
105        Self::new_inner(factory, config, Some(events)).await
106    }
107
108    async fn new_inner(
109        factory: impl ConnectionFactory,
110        config: ReconnectConfig,
111        event_bus: Option<ConnectionEventBus>,
112    ) -> Result<Self, RedisError> {
113        let factory = Arc::new(factory);
114        let conn = match connect_with_timeout(factory.as_ref(), config.connect_timeout).await {
115            Ok(conn) => conn,
116            Err(error) => {
117                if let Some(events) = &event_bus {
118                    events.publish_with(|| ConnectionEvent::ConnectFailed {
119                        error: Arc::from(error.to_string()),
120                    });
121                }
122                return Err(error);
123            }
124        };
125        let inner = match FrameService::from_connection(conn) {
126            Ok(inner) => inner,
127            Err(error) => {
128                if let Some(events) = &event_bus {
129                    events.publish_with(|| ConnectionEvent::ConnectFailed {
130                        error: Arc::from(error.to_string()),
131                    });
132                }
133                return Err(error);
134            }
135        };
136        if let Some(events) = &event_bus {
137            events.publish(ConnectionEvent::Connected);
138        }
139        let disconnect_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
140        let shutdown_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
141        let lifecycle_lock = event_bus.as_ref().map(|_| Arc::new(StdMutex::new(())));
142        Ok(Self {
143            inner,
144            factory,
145            config,
146            state: State::Ready,
147            needs_reconnect: Arc::new(AtomicBool::new(false)),
148            event_bus,
149            disconnect_reported,
150            shutdown_reported,
151            lifecycle_lock,
152        })
153    }
154
155    fn trigger_reconnect(&mut self, attempt: usize, started: Instant) {
156        if self.config.attempt_exhausted(attempt) {
157            if let Some(events) = &self.event_bus {
158                events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
159            }
160            self.state = State::Failed;
161            return;
162        }
163        let delay = self.config.delay_for_attempt(attempt);
164        if let Some(events) = &self.event_bus {
165            events.publish(ConnectionEvent::ReconnectAttempt {
166                attempt: attempt + 1,
167                delay,
168            });
169        }
170        self.state = State::WaitingToReconnect {
171            attempt,
172            sleep: Box::pin(tokio::time::sleep(delay)),
173            started,
174        };
175    }
176}
177
178impl Drop for ReconnectService {
179    fn drop(&mut self) {
180        let _guard = self.lifecycle_lock.as_ref().map(|lock| {
181            lock.lock()
182                .unwrap_or_else(std::sync::PoisonError::into_inner)
183        });
184        publish_shutdown_once(self.event_bus.as_ref(), self.shutdown_reported.as_deref());
185    }
186}
187
188impl Service<Frame> for ReconnectService {
189    type Response = Frame;
190    type Error = RedisError;
191    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
192
193    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
194        // Check if a previous call signaled a connection error.
195        // NOTE: There is a one-request-delay between error detection and
196        // reconnection, because the flag is only checked here in poll_ready.
197        if self.needs_reconnect.swap(false, Ordering::Acquire) && matches!(self.state, State::Ready)
198        {
199            self.trigger_reconnect(0, Instant::now());
200        }
201
202        loop {
203            match &mut self.state {
204                State::Ready => return self.inner.poll_ready(cx),
205                State::Failed => {
206                    return Poll::Ready(Err(RedisError::ReconnectFailed {
207                        attempts: self.config.total_attempt_budget().unwrap_or(0),
208                        last_error: Arc::new(RedisError::ConnectionClosed),
209                    }));
210                }
211                State::WaitingToReconnect {
212                    attempt,
213                    sleep,
214                    started,
215                } => match sleep.as_mut().poll(cx) {
216                    Poll::Ready(()) => {
217                        let attempt = *attempt;
218                        let started = *started;
219                        let factory = Arc::clone(&self.factory);
220                        let connect_timeout = self.config.connect_timeout;
221                        let future: ReconnectFuture = Box::pin(async move {
222                            let conn =
223                                connect_with_timeout(factory.as_ref(), connect_timeout).await?;
224                            FrameService::from_connection(conn)
225                        });
226                        self.state = State::Reconnecting {
227                            attempt,
228                            future,
229                            started,
230                        };
231                    }
232                    Poll::Pending => return Poll::Pending,
233                },
234                State::Reconnecting {
235                    attempt,
236                    future,
237                    started,
238                } => match future.as_mut().poll(cx) {
239                    Poll::Ready(Ok(new_svc)) => {
240                        let attempts = *attempt + 1;
241                        let elapsed = started.elapsed();
242                        self.inner = new_svc;
243                        self.state = State::Ready;
244                        if let Some(disconnect_reported) = &self.disconnect_reported {
245                            disconnect_reported.store(false, Ordering::Release);
246                        }
247                        if let Some(events) = &self.event_bus {
248                            events.publish(ConnectionEvent::Reconnected { attempts, elapsed });
249                        }
250                        return self.inner.poll_ready(cx);
251                    }
252                    Poll::Ready(Err(error)) => {
253                        let next = *attempt + 1;
254                        let started = *started;
255                        if let Some(events) = &self.event_bus {
256                            events.publish_with(|| ConnectionEvent::ReconnectFailed {
257                                attempt: next,
258                                error: Arc::from(error.to_string()),
259                            });
260                        }
261                        self.trigger_reconnect(next, started);
262                    }
263                    Poll::Pending => return Poll::Pending,
264                },
265            }
266        }
267    }
268
269    fn call(&mut self, request: Frame) -> Self::Future {
270        let future = self.inner.call(request);
271        let needs_reconnect = Arc::clone(&self.needs_reconnect);
272        let event_bus = self.event_bus.clone();
273        let disconnect_reported = self.disconnect_reported.clone();
274        let shutdown_reported = self.shutdown_reported.clone();
275        let lifecycle_lock = self.lifecycle_lock.clone();
276
277        Box::pin(async move {
278            let result = future.await;
279            if let Err(ref e) = result
280                && e.is_connection_error()
281            {
282                needs_reconnect.store(true, Ordering::Release);
283                publish_disconnect_before_shutdown(
284                    event_bus.as_ref(),
285                    disconnect_reported.as_deref(),
286                    shutdown_reported.as_deref(),
287                    lifecycle_lock.as_deref(),
288                    e,
289                );
290            }
291            result
292        })
293    }
294}
295
296#[cfg(test)]
297mod tests {
298    use super::*;
299    use std::sync::atomic::AtomicUsize;
300    use std::time::Duration;
301
302    async fn close_after_request(stream: tokio::net::TcpStream) {
303        let mut request = [0; 64];
304        loop {
305            stream.readable().await.unwrap();
306            match stream.try_read(&mut request) {
307                Ok(0) => return,
308                Ok(_) => return,
309                Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => continue,
310                Err(error) => panic!("failed to read mock request: {error}"),
311            }
312        }
313    }
314
315    #[tokio::test]
316    async fn frame_service_events_cover_disconnect_and_reconnect() {
317        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
318        let addr = listener.local_addr().unwrap();
319        let server = tokio::spawn(async move {
320            let (first, _) = listener.accept().await.unwrap();
321            close_after_request(first).await;
322
323            let (second, _) = listener.accept().await.unwrap();
324            futures::future::pending::<()>().await;
325            drop(second);
326        });
327
328        let calls = Arc::new(AtomicUsize::new(0));
329        let factory_calls = Arc::clone(&calls);
330        let factory = move || {
331            let factory_calls = Arc::clone(&factory_calls);
332            async move {
333                factory_calls.fetch_add(1, Ordering::AcqRel);
334                let stream = tokio::net::TcpStream::connect(addr)
335                    .await
336                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
337                Ok(redis_tower_core::RedisConnection::from_stream(
338                    redis_tower_core::RedisStream::Tcp(stream),
339                ))
340            }
341        };
342
343        let events = ConnectionEventBus::new(8);
344        let mut event_stream = events.subscribe();
345        let config = ReconnectConfig {
346            max_retries: Some(2),
347            base_delay: Duration::ZERO,
348            max_delay: Duration::ZERO,
349            jitter: false,
350            connect_timeout: None,
351        };
352        let mut service = ReconnectService::new_with_events(factory, config, events)
353            .await
354            .unwrap();
355        assert_eq!(
356            event_stream.recv().await.unwrap(),
357            ConnectionEvent::Connected
358        );
359
360        futures::future::poll_fn(|cx| service.poll_ready(cx))
361            .await
362            .unwrap();
363        let result = service
364            .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
365            .await;
366        assert!(result.is_err());
367        futures::future::poll_fn(|cx| service.poll_ready(cx))
368            .await
369            .unwrap();
370
371        assert!(matches!(
372            event_stream.recv().await.unwrap(),
373            ConnectionEvent::Disconnected { .. }
374        ));
375        assert_eq!(
376            event_stream.recv().await.unwrap(),
377            ConnectionEvent::ReconnectAttempt {
378                attempt: 1,
379                delay: Duration::ZERO,
380            }
381        );
382        assert!(matches!(
383            event_stream.recv().await.unwrap(),
384            ConnectionEvent::Reconnected { attempts: 1, .. }
385        ));
386        assert_eq!(calls.load(Ordering::Acquire), 2);
387
388        drop(service);
389        assert_eq!(
390            event_stream.recv().await.unwrap(),
391            ConnectionEvent::Disconnected {
392                reason: crate::reconnect::ConnectionDisconnectReason::Shutdown,
393            }
394        );
395
396        server.abort();
397    }
398
399    #[tokio::test]
400    async fn initial_factory_timeout_publishes_connect_failed() {
401        let factory = || async {
402            futures::future::pending::<Result<redis_tower_core::RedisConnection, RedisError>>()
403                .await
404        };
405        let events = ConnectionEventBus::new(4);
406        let mut stream = events.subscribe();
407        let config = ReconnectConfig::default().connect_timeout(Duration::from_millis(10));
408
409        let result = ReconnectService::new_with_events(factory, config, events).await;
410        assert!(matches!(result, Err(RedisError::ConnectTimeout)));
411        assert_eq!(
412            stream.recv().await.unwrap(),
413            ConnectionEvent::ConnectFailed {
414                error: Arc::from(RedisError::ConnectTimeout.to_string()),
415            }
416        );
417    }
418
419    #[tokio::test]
420    async fn reconnect_factory_timeout_exhausts_with_complete_event_order() {
421        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
422        let addr = listener.local_addr().unwrap();
423        let server = tokio::spawn(async move {
424            let (first, _) = listener.accept().await.unwrap();
425            close_after_request(first).await;
426        });
427
428        let calls = Arc::new(AtomicUsize::new(0));
429        let factory_calls = Arc::clone(&calls);
430        let factory = move || {
431            let factory_calls = Arc::clone(&factory_calls);
432            async move {
433                if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
434                    return futures::future::pending::<
435                        Result<redis_tower_core::RedisConnection, RedisError>,
436                    >()
437                    .await;
438                }
439                let stream = tokio::net::TcpStream::connect(addr)
440                    .await
441                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
442                Ok(redis_tower_core::RedisConnection::from_stream(
443                    redis_tower_core::RedisStream::Tcp(stream),
444                ))
445            }
446        };
447
448        let events = ConnectionEventBus::new(8);
449        let mut stream = events.subscribe();
450        let config = ReconnectConfig {
451            max_retries: Some(0),
452            base_delay: Duration::ZERO,
453            max_delay: Duration::ZERO,
454            jitter: false,
455            // Windows' timer granularity can exceed the 10 ms used by the
456            // Linux-only CI that originally covered this path.
457            connect_timeout: Some(Duration::from_millis(100)),
458        };
459        let mut service = ReconnectService::new_with_events(factory, config, events)
460            .await
461            .unwrap();
462        assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
463
464        futures::future::poll_fn(|cx| service.poll_ready(cx))
465            .await
466            .unwrap();
467        assert!(
468            service
469                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
470                .await
471                .is_err()
472        );
473        let reconnect = futures::future::poll_fn(|cx| service.poll_ready(cx)).await;
474        assert!(
475            matches!(
476                reconnect,
477                Err(RedisError::ReconnectFailed { attempts: 1, .. })
478            ),
479            "unexpected reconnect result: {reconnect:?}"
480        );
481
482        assert!(matches!(
483            stream.recv().await.unwrap(),
484            ConnectionEvent::Disconnected { .. }
485        ));
486        assert_eq!(
487            stream.recv().await.unwrap(),
488            ConnectionEvent::ReconnectAttempt {
489                attempt: 1,
490                delay: Duration::ZERO,
491            }
492        );
493        assert_eq!(
494            stream.recv().await.unwrap(),
495            ConnectionEvent::ReconnectFailed {
496                attempt: 1,
497                error: Arc::from(RedisError::ConnectTimeout.to_string()),
498            }
499        );
500        assert_eq!(
501            stream.recv().await.unwrap(),
502            ConnectionEvent::ReconnectExhausted { attempts: 1 }
503        );
504        assert_eq!(calls.load(Ordering::Acquire), 2);
505
506        drop(service);
507        assert_eq!(
508            stream.recv().await.unwrap(),
509            ConnectionEvent::Disconnected {
510                reason: crate::reconnect::ConnectionDisconnectReason::Shutdown,
511            }
512        );
513        server.await.unwrap();
514    }
515
516    #[test]
517    fn trigger_reconnect_transitions_to_failed_when_max_exceeded() {
518        let config = ReconnectConfig::default().max_retries(3);
519        assert!(!config.attempt_exhausted(0));
520        assert!(!config.attempt_exhausted(3));
521        assert!(config.attempt_exhausted(4));
522        assert_eq!(config.total_attempt_budget(), Some(4));
523    }
524
525    #[test]
526    fn infinite_retries_never_transition_to_failed() {
527        let config = ReconnectConfig {
528            max_retries: None,
529            base_delay: Duration::from_millis(10),
530            ..Default::default()
531        };
532        assert!(!config.attempt_exhausted(1_000_000));
533    }
534}