1use std::future::Future;
29use std::pin::Pin;
30use std::sync::Arc;
31use std::sync::Mutex as StdMutex;
32use std::sync::atomic::{AtomicBool, Ordering};
33use std::task::{Context, Poll};
34use std::time::Instant;
35
36use redis_tower_core::{Frame, FrameService, RedisError};
37use tower_service::Service;
38
39use crate::reconnect::{
40 ConnectionEvent, ConnectionEventBus, ConnectionFactory, ReconnectConfig, connect_with_timeout,
41 publish_disconnect_before_shutdown, publish_shutdown_once,
42};
43
44type ReconnectFuture = Pin<Box<dyn Future<Output = Result<FrameService, RedisError>> + Send>>;
45
46enum State {
47 Ready,
48 WaitingToReconnect {
49 attempt: usize,
50 sleep: Pin<Box<tokio::time::Sleep>>,
51 started: Instant,
52 },
53 Reconnecting {
54 attempt: usize,
55 future: ReconnectFuture,
56 started: Instant,
57 },
58 Failed,
59}
60
61pub struct ReconnectService {
75 inner: FrameService,
76 factory: Arc<dyn ConnectionFactory>,
77 config: ReconnectConfig,
78 state: State,
79 needs_reconnect: Arc<AtomicBool>,
80 event_bus: Option<ConnectionEventBus>,
81 disconnect_reported: Option<Arc<AtomicBool>>,
82 shutdown_reported: Option<Arc<AtomicBool>>,
83 lifecycle_lock: Option<Arc<StdMutex<()>>>,
84}
85
86impl ReconnectService {
87 pub async fn new(
89 factory: impl ConnectionFactory,
90 config: ReconnectConfig,
91 ) -> Result<Self, RedisError> {
92 Self::new_inner(factory, config, None).await
93 }
94
95 pub async fn new_with_events(
101 factory: impl ConnectionFactory,
102 config: ReconnectConfig,
103 events: ConnectionEventBus,
104 ) -> Result<Self, RedisError> {
105 Self::new_inner(factory, config, Some(events)).await
106 }
107
108 async fn new_inner(
109 factory: impl ConnectionFactory,
110 config: ReconnectConfig,
111 event_bus: Option<ConnectionEventBus>,
112 ) -> Result<Self, RedisError> {
113 let factory = Arc::new(factory);
114 let conn = match connect_with_timeout(factory.as_ref(), config.connect_timeout).await {
115 Ok(conn) => conn,
116 Err(error) => {
117 if let Some(events) = &event_bus {
118 events.publish_with(|| ConnectionEvent::ConnectFailed {
119 error: Arc::from(error.to_string()),
120 });
121 }
122 return Err(error);
123 }
124 };
125 let inner = match FrameService::from_connection(conn) {
126 Ok(inner) => inner,
127 Err(error) => {
128 if let Some(events) = &event_bus {
129 events.publish_with(|| ConnectionEvent::ConnectFailed {
130 error: Arc::from(error.to_string()),
131 });
132 }
133 return Err(error);
134 }
135 };
136 if let Some(events) = &event_bus {
137 events.publish(ConnectionEvent::Connected);
138 }
139 let disconnect_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
140 let shutdown_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
141 let lifecycle_lock = event_bus.as_ref().map(|_| Arc::new(StdMutex::new(())));
142 Ok(Self {
143 inner,
144 factory,
145 config,
146 state: State::Ready,
147 needs_reconnect: Arc::new(AtomicBool::new(false)),
148 event_bus,
149 disconnect_reported,
150 shutdown_reported,
151 lifecycle_lock,
152 })
153 }
154
155 fn trigger_reconnect(&mut self, attempt: usize, started: Instant) {
156 if self.config.attempt_exhausted(attempt) {
157 if let Some(events) = &self.event_bus {
158 events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
159 }
160 self.state = State::Failed;
161 return;
162 }
163 let delay = self.config.delay_for_attempt(attempt);
164 if let Some(events) = &self.event_bus {
165 events.publish(ConnectionEvent::ReconnectAttempt {
166 attempt: attempt + 1,
167 delay,
168 });
169 }
170 self.state = State::WaitingToReconnect {
171 attempt,
172 sleep: Box::pin(tokio::time::sleep(delay)),
173 started,
174 };
175 }
176}
177
178impl Drop for ReconnectService {
179 fn drop(&mut self) {
180 let _guard = self.lifecycle_lock.as_ref().map(|lock| {
181 lock.lock()
182 .unwrap_or_else(std::sync::PoisonError::into_inner)
183 });
184 publish_shutdown_once(self.event_bus.as_ref(), self.shutdown_reported.as_deref());
185 }
186}
187
188impl Service<Frame> for ReconnectService {
189 type Response = Frame;
190 type Error = RedisError;
191 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
192
193 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
194 if self.needs_reconnect.swap(false, Ordering::Acquire) && matches!(self.state, State::Ready)
198 {
199 self.trigger_reconnect(0, Instant::now());
200 }
201
202 loop {
203 match &mut self.state {
204 State::Ready => return self.inner.poll_ready(cx),
205 State::Failed => {
206 return Poll::Ready(Err(RedisError::ReconnectFailed {
207 attempts: self.config.total_attempt_budget().unwrap_or(0),
208 last_error: Arc::new(RedisError::ConnectionClosed),
209 }));
210 }
211 State::WaitingToReconnect {
212 attempt,
213 sleep,
214 started,
215 } => match sleep.as_mut().poll(cx) {
216 Poll::Ready(()) => {
217 let attempt = *attempt;
218 let started = *started;
219 let factory = Arc::clone(&self.factory);
220 let connect_timeout = self.config.connect_timeout;
221 let future: ReconnectFuture = Box::pin(async move {
222 let conn =
223 connect_with_timeout(factory.as_ref(), connect_timeout).await?;
224 FrameService::from_connection(conn)
225 });
226 self.state = State::Reconnecting {
227 attempt,
228 future,
229 started,
230 };
231 }
232 Poll::Pending => return Poll::Pending,
233 },
234 State::Reconnecting {
235 attempt,
236 future,
237 started,
238 } => match future.as_mut().poll(cx) {
239 Poll::Ready(Ok(new_svc)) => {
240 let attempts = *attempt + 1;
241 let elapsed = started.elapsed();
242 self.inner = new_svc;
243 self.state = State::Ready;
244 if let Some(disconnect_reported) = &self.disconnect_reported {
245 disconnect_reported.store(false, Ordering::Release);
246 }
247 if let Some(events) = &self.event_bus {
248 events.publish(ConnectionEvent::Reconnected { attempts, elapsed });
249 }
250 return self.inner.poll_ready(cx);
251 }
252 Poll::Ready(Err(error)) => {
253 let next = *attempt + 1;
254 let started = *started;
255 if let Some(events) = &self.event_bus {
256 events.publish_with(|| ConnectionEvent::ReconnectFailed {
257 attempt: next,
258 error: Arc::from(error.to_string()),
259 });
260 }
261 self.trigger_reconnect(next, started);
262 }
263 Poll::Pending => return Poll::Pending,
264 },
265 }
266 }
267 }
268
269 fn call(&mut self, request: Frame) -> Self::Future {
270 let future = self.inner.call(request);
271 let needs_reconnect = Arc::clone(&self.needs_reconnect);
272 let event_bus = self.event_bus.clone();
273 let disconnect_reported = self.disconnect_reported.clone();
274 let shutdown_reported = self.shutdown_reported.clone();
275 let lifecycle_lock = self.lifecycle_lock.clone();
276
277 Box::pin(async move {
278 let result = future.await;
279 if let Err(ref e) = result
280 && e.is_connection_error()
281 {
282 needs_reconnect.store(true, Ordering::Release);
283 publish_disconnect_before_shutdown(
284 event_bus.as_ref(),
285 disconnect_reported.as_deref(),
286 shutdown_reported.as_deref(),
287 lifecycle_lock.as_deref(),
288 e,
289 );
290 }
291 result
292 })
293 }
294}
295
296#[cfg(test)]
297mod tests {
298 use super::*;
299 use std::sync::atomic::AtomicUsize;
300 use std::time::Duration;
301
302 async fn close_after_request(stream: tokio::net::TcpStream) {
303 let mut request = [0; 64];
304 loop {
305 stream.readable().await.unwrap();
306 match stream.try_read(&mut request) {
307 Ok(0) => return,
308 Ok(_) => return,
309 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => continue,
310 Err(error) => panic!("failed to read mock request: {error}"),
311 }
312 }
313 }
314
315 #[tokio::test]
316 async fn frame_service_events_cover_disconnect_and_reconnect() {
317 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
318 let addr = listener.local_addr().unwrap();
319 let server = tokio::spawn(async move {
320 let (first, _) = listener.accept().await.unwrap();
321 close_after_request(first).await;
322
323 let (second, _) = listener.accept().await.unwrap();
324 futures::future::pending::<()>().await;
325 drop(second);
326 });
327
328 let calls = Arc::new(AtomicUsize::new(0));
329 let factory_calls = Arc::clone(&calls);
330 let factory = move || {
331 let factory_calls = Arc::clone(&factory_calls);
332 async move {
333 factory_calls.fetch_add(1, Ordering::AcqRel);
334 let stream = tokio::net::TcpStream::connect(addr)
335 .await
336 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
337 Ok(redis_tower_core::RedisConnection::from_stream(
338 redis_tower_core::RedisStream::Tcp(stream),
339 ))
340 }
341 };
342
343 let events = ConnectionEventBus::new(8);
344 let mut event_stream = events.subscribe();
345 let config = ReconnectConfig {
346 max_retries: Some(2),
347 base_delay: Duration::ZERO,
348 max_delay: Duration::ZERO,
349 jitter: false,
350 connect_timeout: None,
351 };
352 let mut service = ReconnectService::new_with_events(factory, config, events)
353 .await
354 .unwrap();
355 assert_eq!(
356 event_stream.recv().await.unwrap(),
357 ConnectionEvent::Connected
358 );
359
360 futures::future::poll_fn(|cx| service.poll_ready(cx))
361 .await
362 .unwrap();
363 let result = service
364 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
365 .await;
366 assert!(result.is_err());
367 futures::future::poll_fn(|cx| service.poll_ready(cx))
368 .await
369 .unwrap();
370
371 assert!(matches!(
372 event_stream.recv().await.unwrap(),
373 ConnectionEvent::Disconnected { .. }
374 ));
375 assert_eq!(
376 event_stream.recv().await.unwrap(),
377 ConnectionEvent::ReconnectAttempt {
378 attempt: 1,
379 delay: Duration::ZERO,
380 }
381 );
382 assert!(matches!(
383 event_stream.recv().await.unwrap(),
384 ConnectionEvent::Reconnected { attempts: 1, .. }
385 ));
386 assert_eq!(calls.load(Ordering::Acquire), 2);
387
388 drop(service);
389 assert_eq!(
390 event_stream.recv().await.unwrap(),
391 ConnectionEvent::Disconnected {
392 reason: crate::reconnect::ConnectionDisconnectReason::Shutdown,
393 }
394 );
395
396 server.abort();
397 }
398
399 #[tokio::test]
400 async fn initial_factory_timeout_publishes_connect_failed() {
401 let factory = || async {
402 futures::future::pending::<Result<redis_tower_core::RedisConnection, RedisError>>()
403 .await
404 };
405 let events = ConnectionEventBus::new(4);
406 let mut stream = events.subscribe();
407 let config = ReconnectConfig::default().connect_timeout(Duration::from_millis(10));
408
409 let result = ReconnectService::new_with_events(factory, config, events).await;
410 assert!(matches!(result, Err(RedisError::ConnectTimeout)));
411 assert_eq!(
412 stream.recv().await.unwrap(),
413 ConnectionEvent::ConnectFailed {
414 error: Arc::from(RedisError::ConnectTimeout.to_string()),
415 }
416 );
417 }
418
419 #[tokio::test]
420 async fn reconnect_factory_timeout_exhausts_with_complete_event_order() {
421 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
422 let addr = listener.local_addr().unwrap();
423 let server = tokio::spawn(async move {
424 let (first, _) = listener.accept().await.unwrap();
425 close_after_request(first).await;
426 });
427
428 let calls = Arc::new(AtomicUsize::new(0));
429 let factory_calls = Arc::clone(&calls);
430 let factory = move || {
431 let factory_calls = Arc::clone(&factory_calls);
432 async move {
433 if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
434 return futures::future::pending::<
435 Result<redis_tower_core::RedisConnection, RedisError>,
436 >()
437 .await;
438 }
439 let stream = tokio::net::TcpStream::connect(addr)
440 .await
441 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
442 Ok(redis_tower_core::RedisConnection::from_stream(
443 redis_tower_core::RedisStream::Tcp(stream),
444 ))
445 }
446 };
447
448 let events = ConnectionEventBus::new(8);
449 let mut stream = events.subscribe();
450 let config = ReconnectConfig {
451 max_retries: Some(0),
452 base_delay: Duration::ZERO,
453 max_delay: Duration::ZERO,
454 jitter: false,
455 connect_timeout: Some(Duration::from_millis(100)),
458 };
459 let mut service = ReconnectService::new_with_events(factory, config, events)
460 .await
461 .unwrap();
462 assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
463
464 futures::future::poll_fn(|cx| service.poll_ready(cx))
465 .await
466 .unwrap();
467 assert!(
468 service
469 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
470 .await
471 .is_err()
472 );
473 let reconnect = futures::future::poll_fn(|cx| service.poll_ready(cx)).await;
474 assert!(
475 matches!(
476 reconnect,
477 Err(RedisError::ReconnectFailed { attempts: 1, .. })
478 ),
479 "unexpected reconnect result: {reconnect:?}"
480 );
481
482 assert!(matches!(
483 stream.recv().await.unwrap(),
484 ConnectionEvent::Disconnected { .. }
485 ));
486 assert_eq!(
487 stream.recv().await.unwrap(),
488 ConnectionEvent::ReconnectAttempt {
489 attempt: 1,
490 delay: Duration::ZERO,
491 }
492 );
493 assert_eq!(
494 stream.recv().await.unwrap(),
495 ConnectionEvent::ReconnectFailed {
496 attempt: 1,
497 error: Arc::from(RedisError::ConnectTimeout.to_string()),
498 }
499 );
500 assert_eq!(
501 stream.recv().await.unwrap(),
502 ConnectionEvent::ReconnectExhausted { attempts: 1 }
503 );
504 assert_eq!(calls.load(Ordering::Acquire), 2);
505
506 drop(service);
507 assert_eq!(
508 stream.recv().await.unwrap(),
509 ConnectionEvent::Disconnected {
510 reason: crate::reconnect::ConnectionDisconnectReason::Shutdown,
511 }
512 );
513 server.await.unwrap();
514 }
515
516 #[test]
517 fn trigger_reconnect_transitions_to_failed_when_max_exceeded() {
518 let config = ReconnectConfig::default().max_retries(3);
519 assert!(!config.attempt_exhausted(0));
520 assert!(!config.attempt_exhausted(3));
521 assert!(config.attempt_exhausted(4));
522 assert_eq!(config.total_attempt_budget(), Some(4));
523 }
524
525 #[test]
526 fn infinite_retries_never_transition_to_failed() {
527 let config = ReconnectConfig {
528 max_retries: None,
529 base_delay: Duration::from_millis(10),
530 ..Default::default()
531 };
532 assert!(!config.attempt_exhausted(1_000_000));
533 }
534}