1use std::collections::VecDeque;
9use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
10use std::sync::{Arc, Mutex as StdMutex};
11use std::time::{Duration, Instant};
12
13use redis_tower_commands::Ping;
14use redis_tower_core::{Command, ConnectionConfig, Frame, RedisConnection, RedisError};
15use tokio::sync::{Mutex, Notify, watch};
16use tower_service::Service;
17
18use crate::circuit_breaker::{RedisCircuitBreakerClient, RedisCircuitBreakerConfig};
19use crate::reconnect::{
20 AddrConnectionFactory, ConnectionDisconnectReason, ConnectionEvent, ConnectionEventBus,
21 ConnectionFactory, ReconnectConfig, UrlConnectionFactory, publish_disconnect_once,
22};
23use crate::retry::{RetryClient, RetryPolicy};
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub struct OfflineQueueConfig {
48 capacity: usize,
49 max_replay_attempts: usize,
50}
51
52impl OfflineQueueConfig {
53 #[must_use]
55 pub const fn new(capacity: usize) -> Self {
56 Self {
57 capacity,
58 max_replay_attempts: 3,
59 }
60 }
61
62 #[must_use]
64 pub const fn capacity(&self) -> usize {
65 self.capacity
66 }
67
68 #[must_use]
75 pub const fn with_max_replay_attempts(mut self, max_replay_attempts: usize) -> Self {
76 assert!(
77 max_replay_attempts > 0,
78 "offline queue replay attempts must be non-zero"
79 );
80 self.max_replay_attempts = max_replay_attempts;
81 self
82 }
83
84 #[must_use]
86 pub const fn max_replay_attempts(&self) -> usize {
87 self.max_replay_attempts
88 }
89}
90
91impl Default for OfflineQueueConfig {
92 fn default() -> Self {
93 Self::new(1024)
94 }
95}
96
97#[derive(Clone)]
176pub struct ResilientRedisClient {
177 conn: Arc<Mutex<RedisConnection>>,
178 factory: Arc<dyn ConnectionFactory>,
179 config: ReconnectConfig,
180 gate: Arc<ReconnectGate>,
183 offline_queue: Option<OfflineQueueHandle>,
185 connection_events: Option<ConnectionEventBus>,
186 disconnect_reported: Option<Arc<AtomicBool>>,
187 _lifecycle: Option<ResilientClientLifecycle>,
190}
191
192impl ResilientRedisClient {
193 pub async fn connect(addr: &str) -> Result<Self, RedisError> {
195 Self::with_config(AddrConnectionFactory::new(addr), ReconnectConfig::default()).await
196 }
197
198 pub async fn connect_with_events(
204 addr: &str,
205 events: ConnectionEventBus,
206 ) -> Result<Self, RedisError> {
207 Self::with_config_and_events(
208 AddrConnectionFactory::new(addr),
209 ReconnectConfig::default(),
210 events,
211 )
212 .await
213 }
214
215 pub async fn connect_with_offline_queue(
220 addr: &str,
221 queue_config: OfflineQueueConfig,
222 ) -> Result<Self, RedisError> {
223 Self::with_config_and_offline_queue(
224 AddrConnectionFactory::new(addr),
225 ReconnectConfig::default(),
226 queue_config,
227 )
228 .await
229 }
230
231 pub async fn connect_with_offline_queue_and_events(
233 addr: &str,
234 queue_config: OfflineQueueConfig,
235 events: ConnectionEventBus,
236 ) -> Result<Self, RedisError> {
237 Self::with_config_and_offline_queue_and_events(
238 AddrConnectionFactory::new(addr),
239 ReconnectConfig::default(),
240 queue_config,
241 events,
242 )
243 .await
244 }
245
246 pub async fn connect_with_connection_config(
248 addr: &str,
249 connection_config: ConnectionConfig,
250 ) -> Result<Self, RedisError> {
251 Self::with_config(
252 AddrConnectionFactory::new(addr).with_connection_config(connection_config),
253 ReconnectConfig::default(),
254 )
255 .await
256 }
257
258 pub async fn connect_url(url: &str) -> Result<Self, RedisError> {
260 Self::with_config(UrlConnectionFactory::new(url), ReconnectConfig::default()).await
261 }
262
263 pub async fn connect_url_with_offline_queue(
265 url: &str,
266 queue_config: OfflineQueueConfig,
267 ) -> Result<Self, RedisError> {
268 Self::with_config_and_offline_queue(
269 UrlConnectionFactory::new(url),
270 ReconnectConfig::default(),
271 queue_config,
272 )
273 .await
274 }
275
276 pub async fn connect_url_with_connection_config(
278 url: &str,
279 connection_config: ConnectionConfig,
280 ) -> Result<Self, RedisError> {
281 Self::with_config(
282 UrlConnectionFactory::new(url).with_connection_config(connection_config),
283 ReconnectConfig::default(),
284 )
285 .await
286 }
287
288 pub async fn with_config(
290 factory: impl ConnectionFactory,
291 config: ReconnectConfig,
292 ) -> Result<Self, RedisError> {
293 Self::build(factory, config, None, None).await
294 }
295
296 pub async fn with_config_and_events(
298 factory: impl ConnectionFactory,
299 config: ReconnectConfig,
300 events: ConnectionEventBus,
301 ) -> Result<Self, RedisError> {
302 Self::build(factory, config, None, Some(events)).await
303 }
304
305 pub async fn with_config_and_offline_queue(
311 factory: impl ConnectionFactory,
312 config: ReconnectConfig,
313 queue_config: OfflineQueueConfig,
314 ) -> Result<Self, RedisError> {
315 Self::build(factory, config, Some(queue_config), None).await
316 }
317
318 pub async fn with_config_and_offline_queue_and_events(
320 factory: impl ConnectionFactory,
321 config: ReconnectConfig,
322 queue_config: OfflineQueueConfig,
323 events: ConnectionEventBus,
324 ) -> Result<Self, RedisError> {
325 Self::build(factory, config, Some(queue_config), Some(events)).await
326 }
327
328 async fn build(
329 factory: impl ConnectionFactory,
330 config: ReconnectConfig,
331 queue_config: Option<OfflineQueueConfig>,
332 connection_events: Option<ConnectionEventBus>,
333 ) -> Result<Self, RedisError> {
334 let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
335 let conn = match connect_with_timeout(&*factory, config.connect_timeout).await {
336 Ok(conn) => conn,
337 Err(error) => {
338 if let Some(events) = &connection_events {
339 events.publish_with(|| ConnectionEvent::ConnectFailed {
340 error: Arc::from(error.to_string()),
341 });
342 }
343 return Err(error);
344 }
345 };
346 if let Some(events) = &connection_events {
347 events.publish(ConnectionEvent::Connected);
348 }
349 let disconnect_reported = connection_events
350 .as_ref()
351 .map(|_| Arc::new(AtomicBool::new(false)));
352 let lifecycle = connection_events
353 .as_ref()
354 .map(|events| ResilientClientLifecycle::new(events.clone()));
355 Ok(Self {
356 conn: Arc::new(Mutex::new(conn)),
357 factory,
358 config,
359 gate: Arc::new(ReconnectGate::new()),
360 offline_queue: queue_config.map(OfflineQueueHandle::new),
361 connection_events,
362 disconnect_reported,
363 _lifecycle: lifecycle,
364 })
365 }
366
367 pub async fn execute<Cmd: Command>(&self, cmd: Cmd) -> Result<Cmd::Response, RedisError> {
383 if let Some(queue) = &self.offline_queue {
384 let deadline = cmd.deadline();
385 if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
386 return Err(RedisError::CommandTimeout);
387 }
388 let operation = self.execute_with_offline_queue(cmd, queue);
389 return match deadline {
390 Some(deadline) => tokio::time::timeout_at(deadline, operation)
391 .await
392 .map_err(|_elapsed| RedisError::CommandTimeout)?,
393 None => operation.await,
394 };
395 }
396
397 let mut conn = self.conn.lock().await;
398 let result = conn.execute(cmd).await;
399
400 if let Err(ref e) = result
401 && e.is_connection_error()
402 {
403 self.publish_disconnect(e);
404 drop(conn);
405 self.reconnect().await;
406 }
407
408 result
409 }
410
411 #[must_use]
416 pub fn offline_queue_depth(&self) -> usize {
417 self.offline_queue
418 .as_ref()
419 .map_or(0, OfflineQueueHandle::depth)
420 }
421
422 #[must_use]
424 pub fn is_reconnecting(&self) -> bool {
425 self.offline_queue
426 .as_ref()
427 .is_some_and(OfflineQueueHandle::is_reconnecting)
428 || self.gate.is_reconnecting()
429 }
430
431 async fn execute_with_offline_queue<Cmd: Command>(
432 &self,
433 mut cmd: Cmd,
434 queue: &OfflineQueueHandle,
435 ) -> Result<Cmd::Response, RedisError> {
436 let idempotent = cmd.idempotent();
437 let frame = cmd.to_frame();
440
441 match queue.before_execute(idempotent) {
442 QueueDecision::Execute => {}
443 QueueDecision::Wait { permit, campaign } => {
444 if let Some(campaign) = campaign {
445 self.spawn_reconnect_campaign(queue, campaign);
446 }
447 return self.execute_queued(cmd, frame, queue, permit).await;
448 }
449 QueueDecision::Reject { error, campaign } => {
450 if let Some(campaign) = campaign {
451 self.spawn_reconnect_campaign(queue, campaign);
452 }
453 return Err(error.into_redis_error());
454 }
455 }
456
457 let mut conn = self.conn.lock().await;
458
459 match queue.before_execute(idempotent) {
463 QueueDecision::Execute => {}
464 QueueDecision::Wait { permit, campaign } => {
465 drop(conn);
466 if let Some(campaign) = campaign {
467 self.spawn_reconnect_campaign(queue, campaign);
468 }
469 return self.execute_queued(cmd, frame, queue, permit).await;
470 }
471 QueueDecision::Reject { error, campaign } => {
472 drop(conn);
473 if let Some(campaign) = campaign {
474 self.spawn_reconnect_campaign(queue, campaign);
475 }
476 return Err(error.into_redis_error());
477 }
478 }
479
480 let mut alignment = ProtocolAlignmentGuard::new(self, queue);
481 let (returned_cmd, result) = execute_retained(&mut conn, cmd, frame.clone()).await;
482 alignment.disarm();
483 cmd = returned_cmd;
484 let Err(error) = result else {
485 return result;
486 };
487 if !error.is_connection_error() {
488 return Err(error);
489 }
490
491 self.publish_disconnect(&error);
492 let decision = queue.after_connection_error(idempotent);
493 drop(conn);
494 match decision {
495 QueueDecision::Wait { permit, campaign } => {
496 if let Some(campaign) = campaign {
497 self.spawn_reconnect_campaign(queue, campaign);
498 }
499 self.execute_queued(cmd, frame, queue, permit).await
500 }
501 QueueDecision::Reject {
502 error: rejection,
503 campaign,
504 } => {
505 if let Some(campaign) = campaign {
506 self.spawn_reconnect_campaign(queue, campaign);
507 }
508 if matches!(rejection, QueueRejection::Full) {
512 Err(RedisError::QueueFull)
513 } else {
514 Err(error)
515 }
516 }
517 QueueDecision::Execute => unreachable!("a disconnect must leave offline mode"),
518 }
519 }
520
521 async fn execute_queued<Cmd: Command>(
522 &self,
523 mut cmd: Cmd,
524 frame: Frame,
525 queue: &OfflineQueueHandle,
526 permit: QueuePermit,
527 ) -> Result<Cmd::Response, RedisError> {
528 let mut replay_attempts = 0;
529 loop {
530 permit.wait_for_turn().await?;
531 let mut conn = self.conn.lock().await;
532
533 match permit.check_turn() {
538 TurnState::Ready => {}
539 TurnState::Wait => {
540 drop(conn);
541 continue;
542 }
543 TurnState::Rejected(error) => return Err(error.into_redis_error()),
544 }
545
546 let mut alignment = ProtocolAlignmentGuard::new(self, queue);
547 let (returned_cmd, result) = execute_retained(&mut conn, cmd, frame.clone()).await;
548 alignment.disarm();
549 cmd = returned_cmd;
550 let Err(error) = result else {
551 return result;
552 };
553 if !error.is_connection_error() {
554 return Err(error);
555 }
556
557 self.publish_disconnect(&error);
558 let campaign = queue.reconnect_existing(&permit);
559 replay_attempts += 1;
560 drop(conn);
561 if let Some(campaign) = campaign {
562 self.spawn_reconnect_campaign(queue, campaign);
563 }
564 if replay_attempts >= queue.max_replay_attempts() {
565 return Err(RedisError::ReconnectFailed {
566 attempts: replay_attempts,
567 last_error: Arc::new(error),
568 });
569 }
570 }
574 }
575
576 fn spawn_reconnect_campaign(&self, queue: &OfflineQueueHandle, campaign: u64) {
577 let factory = Arc::clone(&self.factory);
578 let config = self.config.clone();
579 let conn = Arc::downgrade(&self.conn);
580 let queue = Arc::downgrade(&queue.inner);
581 let lifecycle = self
582 ._lifecycle
583 .as_ref()
584 .map(ResilientClientLifecycle::control);
585 let disconnect_reported = self.disconnect_reported.clone();
586 let mut closed = queue
587 .upgrade()
588 .expect("offline queue owner exists while starting reconnect")
589 .closed_tx
590 .subscribe();
591
592 tokio::spawn(async move {
593 let outcome = reconnect_campaign_until_closed(
594 &*factory,
595 &config,
596 &mut closed,
597 lifecycle.as_deref(),
598 )
599 .await;
600 let Some(queue) = queue.upgrade() else {
601 return;
602 };
603
604 match outcome {
605 CancellableReconnect::Connected {
606 connection: new_conn,
607 attempts,
608 elapsed,
609 } => {
610 let Some(conn) = conn.upgrade() else {
611 return;
612 };
613 let lock = conn.lock();
614 tokio::pin!(lock);
615 let mut conn = tokio::select! {
616 result = closed.changed() => {
617 let _ = result;
618 return;
619 }
620 conn = &mut lock => conn,
621 };
622 if queue.campaign_is_active(campaign) {
623 *conn = new_conn;
624 if let Some(reported) = &disconnect_reported {
625 reported.store(false, Ordering::Release);
626 }
627 if let Some(lifecycle) = &lifecycle {
628 lifecycle.publish(ConnectionEvent::Reconnected { attempts, elapsed });
629 }
630 queue.finish_reconnect(campaign, None);
631 }
632 }
633 CancellableReconnect::Failed {
634 attempts,
635 last_error,
636 } => {
637 queue.finish_reconnect(
638 campaign,
639 Some(ReconnectFailure {
640 attempts,
641 last_error: Arc::new(last_error),
642 }),
643 );
644 }
645 CancellableReconnect::Closed => {}
646 }
647 });
648 }
649
650 fn publish_disconnect(&self, error: &RedisError) {
651 publish_disconnect_once(
652 self.connection_events.as_ref(),
653 self.disconnect_reported.as_deref(),
654 error,
655 );
656 }
657
658 pub async fn health_check(&self) -> Result<(), RedisError> {
663 self.execute(Ping::new()).await?;
664 Ok(())
665 }
666
667 pub fn retry(&self, policy: RetryPolicy) -> RetryClient<Self> {
692 RetryClient::new(self.clone(), policy)
693 }
694
695 pub fn with_circuit_breaker(
702 self,
703 config: RedisCircuitBreakerConfig,
704 ) -> RedisCircuitBreakerClient<Self> {
705 RedisCircuitBreakerClient::new(self, config)
706 }
707
708 async fn reconnect(&self) {
710 let seen = self.gate.generation();
714 let _guard = self.gate.enter().await;
715 if self.gate.generation() != seen {
716 return;
717 }
718
719 let _active = self.gate.mark_reconnecting();
720 let lifecycle = self._lifecycle.as_ref().map(|lease| lease.control.as_ref());
721 if let Some(success) = reconnect_campaign(&*self.factory, &self.config, lifecycle).await {
722 *self.conn.lock().await = success.connection;
723 if let Some(reported) = &self.disconnect_reported {
724 reported.store(false, Ordering::Release);
725 }
726 if let Some(lifecycle) = lifecycle {
727 lifecycle.publish(ConnectionEvent::Reconnected {
728 attempts: success.attempts,
729 elapsed: success.elapsed,
730 });
731 }
732 self.gate.mark_reconnected();
733 }
734 }
735}
736
737struct ResilientLifecycleControl {
738 state: StdMutex<ResilientLifecycleState>,
739 events: ConnectionEventBus,
740}
741
742struct ResilientLifecycleState {
743 handles: usize,
744 shutdown_published: bool,
745}
746
747impl ResilientLifecycleControl {
748 fn new(events: ConnectionEventBus) -> Self {
749 Self {
750 state: StdMutex::new(ResilientLifecycleState {
751 handles: 1,
752 shutdown_published: false,
753 }),
754 events,
755 }
756 }
757
758 fn state(&self) -> std::sync::MutexGuard<'_, ResilientLifecycleState> {
759 self.state
760 .lock()
761 .unwrap_or_else(std::sync::PoisonError::into_inner)
762 }
763
764 fn add_handle(&self) {
765 let mut state = self.state();
766 state.handles = state
767 .handles
768 .checked_add(1)
769 .expect("ResilientRedisClient handle count overflowed");
770 }
771
772 fn release_handle(&self) {
773 let mut state = self.state();
774 state.handles = state
775 .handles
776 .checked_sub(1)
777 .expect("ResilientRedisClient handle count underflowed");
778 if state.handles == 0 && !state.shutdown_published {
779 state.shutdown_published = true;
780 self.events.publish(ConnectionEvent::Disconnected {
781 reason: ConnectionDisconnectReason::Shutdown,
782 });
783 }
784 }
785
786 fn publish_with(&self, make_event: impl FnOnce() -> ConnectionEvent) -> bool {
790 let state = self.state();
791 if state.handles == 0 {
792 return false;
793 }
794 self.events.publish_with(make_event);
795 true
796 }
797
798 fn publish(&self, event: ConnectionEvent) -> bool {
799 self.publish_with(|| event)
800 }
801}
802
803struct ResilientClientLifecycle {
804 control: Arc<ResilientLifecycleControl>,
805}
806
807impl ResilientClientLifecycle {
808 fn new(events: ConnectionEventBus) -> Self {
809 Self {
810 control: Arc::new(ResilientLifecycleControl::new(events)),
811 }
812 }
813
814 fn control(&self) -> Arc<ResilientLifecycleControl> {
815 Arc::clone(&self.control)
816 }
817}
818
819impl Clone for ResilientClientLifecycle {
820 fn clone(&self) -> Self {
821 self.control.add_handle();
822 Self {
823 control: Arc::clone(&self.control),
824 }
825 }
826}
827
828impl Drop for ResilientClientLifecycle {
829 fn drop(&mut self) {
830 self.control.release_handle();
831 }
832}
833
834impl Service<Frame> for ResilientRedisClient {
835 type Response = Frame;
836 type Error = RedisError;
837 type Future =
838 std::pin::Pin<Box<dyn std::future::Future<Output = Result<Frame, RedisError>> + Send>>;
839
840 fn poll_ready(
841 &mut self,
842 _cx: &mut std::task::Context<'_>,
843 ) -> std::task::Poll<Result<(), Self::Error>> {
844 std::task::Poll::Ready(Ok(()))
845 }
846
847 fn call(&mut self, request: Frame) -> Self::Future {
848 let client = self.clone();
849 Box::pin(async move {
850 if let Some(queue) = &client.offline_queue {
851 match queue.before_execute(false) {
852 QueueDecision::Execute => {}
853 QueueDecision::Reject { error, campaign } => {
854 if let Some(campaign) = campaign {
855 client.spawn_reconnect_campaign(queue, campaign);
856 }
857 return Err(error.into_redis_error());
858 }
859 QueueDecision::Wait { .. } => {
860 unreachable!("raw frames are never admitted to the offline queue")
861 }
862 }
863 }
864
865 let mut connection = client.conn.lock().await;
866
867 if let Some(queue) = &client.offline_queue {
868 match queue.before_execute(false) {
869 QueueDecision::Execute => {}
870 QueueDecision::Reject { error, campaign } => {
871 drop(connection);
872 if let Some(campaign) = campaign {
873 client.spawn_reconnect_campaign(queue, campaign);
874 }
875 return Err(error.into_redis_error());
876 }
877 QueueDecision::Wait { .. } => {
878 unreachable!("raw frames are never admitted to the offline queue")
879 }
880 }
881 }
882
883 let mut alignment = client
884 .offline_queue
885 .as_ref()
886 .map(|queue| ProtocolAlignmentGuard::new(&client, queue));
887 let result =
888 connection
889 .execute_pipeline(vec![request])
890 .await
891 .and_then(|mut responses| {
892 responses.pop().ok_or(RedisError::UnexpectedResponse {
893 expected: "one pipeline response",
894 actual: "empty response".to_string(),
895 })
896 });
897 if let Some(alignment) = &mut alignment {
898 alignment.disarm();
899 }
900
901 if matches!(&result, Err(error) if error.is_connection_error()) {
902 if let Err(error) = &result {
903 client.publish_disconnect(error);
904 }
905 drop(connection);
906 if let Some(queue) = &client.offline_queue {
907 match queue.after_connection_error(false) {
908 QueueDecision::Reject { campaign, .. } => {
909 if let Some(campaign) = campaign {
910 client.spawn_reconnect_campaign(queue, campaign);
911 }
912 }
913 QueueDecision::Execute | QueueDecision::Wait { .. } => {
914 unreachable!("raw frames are never admitted to the offline queue")
915 }
916 }
917 } else {
918 client.reconnect().await;
919 }
920 }
921 result
922 })
923 }
924}
925
926struct ProtocolAlignmentGuard<'a> {
930 client: &'a ResilientRedisClient,
931 queue: &'a OfflineQueueHandle,
932 armed: bool,
933}
934
935impl<'a> ProtocolAlignmentGuard<'a> {
936 fn new(client: &'a ResilientRedisClient, queue: &'a OfflineQueueHandle) -> Self {
937 Self {
938 client,
939 queue,
940 armed: true,
941 }
942 }
943
944 fn disarm(&mut self) {
945 self.armed = false;
946 }
947}
948
949impl Drop for ProtocolAlignmentGuard<'_> {
950 fn drop(&mut self) {
951 if !self.armed {
952 return;
953 }
954
955 self.client
956 .publish_disconnect(&RedisError::ConnectionClosed);
957 if let Some(campaign) = self.queue.taint_connection() {
958 self.client.spawn_reconnect_campaign(self.queue, campaign);
959 }
960 }
961}
962
963async fn execute_retained<Cmd: Command>(
967 conn: &mut RedisConnection,
968 cmd: Cmd,
969 frame: Frame,
970) -> (Cmd, Result<Cmd::Response, RedisError>) {
971 let result = match conn.execute_pipeline(vec![frame]).await {
972 Ok(mut responses) => match responses.pop() {
973 Some(Frame::Error(error)) => Err(RedisError::Redis(
974 String::from_utf8_lossy(&error).into_owned(),
975 )),
976 Some(response) => cmd.parse_response(response),
977 None => Err(RedisError::UnexpectedResponse {
978 expected: "one command response",
979 actual: "empty response".to_string(),
980 }),
981 },
982 Err(error) => Err(error),
983 };
984 (cmd, result)
985}
986
987struct OfflineQueue {
989 capacity: usize,
990 max_replay_attempts: usize,
991 owners: AtomicUsize,
992 state: StdMutex<OfflineQueueState>,
993 changed: Notify,
994 closed_tx: watch::Sender<bool>,
995}
996
997struct OfflineQueueState {
998 phase: OfflinePhase,
999 tickets: VecDeque<u64>,
1000 next_ticket: u64,
1001 next_campaign: u64,
1002 last_reconnect_failure: Option<ReconnectFailure>,
1003}
1004
1005#[derive(Debug, Clone)]
1006struct ReconnectFailure {
1007 attempts: usize,
1008 last_error: Arc<RedisError>,
1009}
1010
1011impl ReconnectFailure {
1012 fn rejection(&self) -> QueueRejection {
1013 QueueRejection::ReconnectFailed {
1014 attempts: self.attempts,
1015 last_error: Arc::clone(&self.last_error),
1016 }
1017 }
1018}
1019
1020#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1021enum OfflinePhase {
1022 Connected,
1023 Reconnecting { campaign: u64 },
1024 Draining,
1025 Failed { attempts: usize },
1026 Closed,
1027}
1028
1029struct OfflineQueueHandle {
1033 inner: Arc<OfflineQueue>,
1034}
1035
1036impl OfflineQueueHandle {
1037 fn new(config: OfflineQueueConfig) -> Self {
1038 let (closed_tx, _) = watch::channel(false);
1039 Self {
1040 inner: Arc::new(OfflineQueue {
1041 capacity: config.capacity,
1042 max_replay_attempts: config.max_replay_attempts,
1043 owners: AtomicUsize::new(1),
1044 state: StdMutex::new(OfflineQueueState {
1045 phase: OfflinePhase::Connected,
1046 tickets: VecDeque::new(),
1047 next_ticket: 0,
1048 next_campaign: 0,
1049 last_reconnect_failure: None,
1050 }),
1051 changed: Notify::new(),
1052 closed_tx,
1053 }),
1054 }
1055 }
1056
1057 fn state(&self) -> std::sync::MutexGuard<'_, OfflineQueueState> {
1058 self.inner
1059 .state
1060 .lock()
1061 .expect("offline queue state mutex poisoned")
1062 }
1063
1064 fn depth(&self) -> usize {
1065 self.state().tickets.len()
1066 }
1067
1068 fn is_reconnecting(&self) -> bool {
1069 matches!(self.state().phase, OfflinePhase::Reconnecting { .. })
1070 }
1071
1072 fn max_replay_attempts(&self) -> usize {
1073 self.inner.max_replay_attempts
1074 }
1075
1076 fn before_execute(&self, idempotent: bool) -> QueueDecision {
1077 let mut state = self.state();
1078 match state.phase {
1079 OfflinePhase::Connected => QueueDecision::Execute,
1080 OfflinePhase::Reconnecting { .. } | OfflinePhase::Draining => {
1081 if idempotent {
1082 self.enqueue_locked(&mut state, None)
1083 } else {
1084 QueueDecision::reject(QueueRejection::ConnectionClosed, None)
1085 }
1086 }
1087 OfflinePhase::Failed { attempts } if !state.tickets.is_empty() => {
1088 if idempotent {
1089 let rejection = state.last_reconnect_failure.as_ref().map_or_else(
1090 || QueueRejection::ReconnectFailed {
1091 attempts,
1092 last_error: Arc::new(RedisError::ConnectionClosed),
1093 },
1094 ReconnectFailure::rejection,
1095 );
1096 QueueDecision::reject(rejection, None)
1097 } else {
1098 QueueDecision::reject(QueueRejection::ConnectionClosed, None)
1099 }
1100 }
1101 OfflinePhase::Failed { .. } => {
1102 let campaign = Self::begin_campaign_locked(&mut state);
1103 if idempotent {
1104 self.enqueue_locked(&mut state, Some(campaign))
1105 } else {
1106 QueueDecision::reject(QueueRejection::ConnectionClosed, Some(campaign))
1107 }
1108 }
1109 OfflinePhase::Closed => QueueDecision::reject(QueueRejection::ConnectionClosed, None),
1110 }
1111 }
1112
1113 fn after_connection_error(&self, idempotent: bool) -> QueueDecision {
1114 let mut state = self.state();
1115 let campaign = match state.phase {
1116 OfflinePhase::Connected | OfflinePhase::Draining | OfflinePhase::Failed { .. } => {
1117 Some(Self::begin_campaign_locked(&mut state))
1118 }
1119 OfflinePhase::Reconnecting { .. } => None,
1120 OfflinePhase::Closed => {
1121 return QueueDecision::reject(QueueRejection::ConnectionClosed, None);
1122 }
1123 };
1124
1125 if idempotent {
1126 self.enqueue_locked(&mut state, campaign)
1127 } else {
1128 QueueDecision::reject(QueueRejection::ConnectionClosed, campaign)
1129 }
1130 }
1131
1132 fn enqueue_locked(
1133 &self,
1134 state: &mut OfflineQueueState,
1135 campaign: Option<u64>,
1136 ) -> QueueDecision {
1137 if state.tickets.len() >= self.inner.capacity {
1138 return QueueDecision::reject(QueueRejection::Full, campaign);
1139 }
1140 let ticket = state.next_ticket;
1141 state.next_ticket = state.next_ticket.wrapping_add(1);
1142 state.tickets.push_back(ticket);
1143 QueueDecision::Wait {
1144 permit: QueuePermit {
1145 queue: Arc::clone(&self.inner),
1146 ticket,
1147 },
1148 campaign,
1149 }
1150 }
1151
1152 fn begin_campaign_locked(state: &mut OfflineQueueState) -> u64 {
1153 let campaign = state.next_campaign;
1154 state.next_campaign = state.next_campaign.wrapping_add(1);
1155 state.phase = OfflinePhase::Reconnecting { campaign };
1156 state.last_reconnect_failure = None;
1157 campaign
1158 }
1159
1160 fn reconnect_existing(&self, permit: &QueuePermit) -> Option<u64> {
1161 let mut state = self.state();
1162 if matches!(state.phase, OfflinePhase::Closed) {
1163 return None;
1164 }
1165 if matches!(state.phase, OfflinePhase::Reconnecting { .. }) {
1166 return None;
1167 }
1168 debug_assert_eq!(state.tickets.front(), Some(&permit.ticket));
1169 Some(Self::begin_campaign_locked(&mut state))
1170 }
1171
1172 fn taint_connection(&self) -> Option<u64> {
1175 let mut state = self.state();
1176 match state.phase {
1177 OfflinePhase::Connected | OfflinePhase::Draining | OfflinePhase::Failed { .. } => {
1178 Some(Self::begin_campaign_locked(&mut state))
1179 }
1180 OfflinePhase::Reconnecting { .. } | OfflinePhase::Closed => None,
1181 }
1182 }
1183}
1184
1185impl Clone for OfflineQueueHandle {
1186 fn clone(&self) -> Self {
1187 self.inner.owners.fetch_add(1, Ordering::Relaxed);
1188 Self {
1189 inner: Arc::clone(&self.inner),
1190 }
1191 }
1192}
1193
1194impl Drop for OfflineQueueHandle {
1195 fn drop(&mut self) {
1196 if self.inner.owners.fetch_sub(1, Ordering::AcqRel) == 1 {
1197 self.inner.close();
1198 }
1199 }
1200}
1201
1202impl OfflineQueue {
1203 fn state(&self) -> std::sync::MutexGuard<'_, OfflineQueueState> {
1204 self.state
1205 .lock()
1206 .expect("offline queue state mutex poisoned")
1207 }
1208
1209 fn close(&self) {
1210 self.state().phase = OfflinePhase::Closed;
1211 self.closed_tx.send_replace(true);
1212 self.changed.notify_waiters();
1213 }
1214
1215 fn campaign_is_active(&self, campaign: u64) -> bool {
1216 matches!(
1217 self.state().phase,
1218 OfflinePhase::Reconnecting { campaign: active } if active == campaign
1219 )
1220 }
1221
1222 fn finish_reconnect(&self, campaign: u64, failure: Option<ReconnectFailure>) {
1223 let mut state = self.state();
1224 if !matches!(
1225 state.phase,
1226 OfflinePhase::Reconnecting { campaign: active } if active == campaign
1227 ) {
1228 return;
1229 }
1230 state.phase = match &failure {
1231 Some(failure) => OfflinePhase::Failed {
1232 attempts: failure.attempts,
1233 },
1234 None if state.tickets.is_empty() => OfflinePhase::Connected,
1235 None => OfflinePhase::Draining,
1236 };
1237 state.last_reconnect_failure = failure;
1238 drop(state);
1239 self.changed.notify_waiters();
1240 }
1241}
1242
1243enum QueueDecision {
1244 Execute,
1245 Wait {
1246 permit: QueuePermit,
1247 campaign: Option<u64>,
1248 },
1249 Reject {
1250 error: QueueRejection,
1251 campaign: Option<u64>,
1252 },
1253}
1254
1255impl QueueDecision {
1256 fn reject(error: QueueRejection, campaign: Option<u64>) -> Self {
1257 Self::Reject { error, campaign }
1258 }
1259}
1260
1261#[derive(Debug, Clone)]
1262enum QueueRejection {
1263 ConnectionClosed,
1264 Full,
1265 ReconnectFailed {
1266 attempts: usize,
1267 last_error: Arc<RedisError>,
1268 },
1269}
1270
1271impl QueueRejection {
1272 fn into_redis_error(self) -> RedisError {
1273 match self {
1274 Self::ConnectionClosed => RedisError::ConnectionClosed,
1275 Self::Full => RedisError::QueueFull,
1276 Self::ReconnectFailed {
1277 attempts,
1278 last_error,
1279 } => RedisError::ReconnectFailed {
1280 attempts,
1281 last_error,
1282 },
1283 }
1284 }
1285}
1286
1287struct QueuePermit {
1288 queue: Arc<OfflineQueue>,
1289 ticket: u64,
1290}
1291
1292impl QueuePermit {
1293 async fn wait_for_turn(&self) -> Result<(), RedisError> {
1294 loop {
1295 let notified = self.queue.changed.notified();
1296 match self.check_turn() {
1297 TurnState::Ready => return Ok(()),
1298 TurnState::Wait => notified.await,
1299 TurnState::Rejected(error) => return Err(error.into_redis_error()),
1300 }
1301 }
1302 }
1303
1304 fn check_turn(&self) -> TurnState {
1305 let state = self.queue.state();
1306 match state.phase {
1307 OfflinePhase::Connected | OfflinePhase::Draining
1308 if state.tickets.front() == Some(&self.ticket) =>
1309 {
1310 TurnState::Ready
1311 }
1312 OfflinePhase::Failed { attempts } => {
1313 let rejection = state.last_reconnect_failure.as_ref().map_or_else(
1314 || QueueRejection::ReconnectFailed {
1315 attempts,
1316 last_error: Arc::new(RedisError::ConnectionClosed),
1317 },
1318 ReconnectFailure::rejection,
1319 );
1320 TurnState::Rejected(rejection)
1321 }
1322 OfflinePhase::Closed => TurnState::Rejected(QueueRejection::ConnectionClosed),
1323 _ => TurnState::Wait,
1324 }
1325 }
1326}
1327
1328impl Drop for QueuePermit {
1329 fn drop(&mut self) {
1330 let mut state = self.queue.state();
1331 let was_front = state.tickets.front() == Some(&self.ticket);
1332 if let Some(index) = state
1333 .tickets
1334 .iter()
1335 .position(|ticket| *ticket == self.ticket)
1336 {
1337 state.tickets.remove(index);
1338 }
1339 if state.tickets.is_empty() && matches!(state.phase, OfflinePhase::Draining) {
1340 state.phase = OfflinePhase::Connected;
1341 }
1342 drop(state);
1343 if was_front {
1344 self.queue.changed.notify_waiters();
1345 }
1346 }
1347}
1348
1349enum TurnState {
1350 Ready,
1351 Wait,
1352 Rejected(QueueRejection),
1353}
1354
1355async fn connect_with_timeout(
1358 factory: &dyn ConnectionFactory,
1359 connect_timeout: Option<Duration>,
1360) -> Result<RedisConnection, RedisError> {
1361 match connect_timeout {
1362 Some(t) => tokio::time::timeout(t, factory.connect())
1363 .await
1364 .map_err(|_| RedisError::ConnectTimeout)?,
1365 None => factory.connect().await,
1366 }
1367}
1368
1369async fn reconnect_campaign(
1374 factory: &dyn ConnectionFactory,
1375 config: &ReconnectConfig,
1376 lifecycle: Option<&ResilientLifecycleControl>,
1377) -> Option<ReconnectSuccess> {
1378 let max = config.max_retries.unwrap_or(usize::MAX);
1379 let started = Instant::now();
1380 for attempt in 0..=max {
1381 let delay = config.delay_for_attempt(attempt);
1382 tracing::warn!(attempt, delay = ?delay, "redis: backing off before reconnect");
1383 if let Some(lifecycle) = lifecycle
1384 && !lifecycle.publish(ConnectionEvent::ReconnectAttempt {
1385 attempt: attempt + 1,
1386 delay,
1387 })
1388 {
1389 return None;
1390 }
1391 if !delay.is_zero() {
1392 tokio::time::sleep(delay).await;
1393 }
1394 match connect_with_timeout(factory, config.connect_timeout).await {
1395 Ok(conn) => {
1396 tracing::info!(attempt, "redis: reconnected successfully");
1397 return Some(ReconnectSuccess {
1398 connection: conn,
1399 attempts: attempt + 1,
1400 elapsed: started.elapsed(),
1401 });
1402 }
1403 Err(e) => {
1404 tracing::warn!(attempt, error = %e, "redis: reconnect attempt failed");
1405 if let Some(lifecycle) = lifecycle
1406 && !lifecycle.publish_with(|| ConnectionEvent::ReconnectFailed {
1407 attempt: attempt + 1,
1408 error: Arc::from(e.to_string()),
1409 })
1410 {
1411 return None;
1412 }
1413 }
1414 }
1415 }
1416 if let Some(lifecycle) = lifecycle {
1417 lifecycle.publish(ConnectionEvent::ReconnectExhausted {
1418 attempts: max.saturating_add(1),
1419 });
1420 }
1421 None
1422}
1423
1424struct ReconnectSuccess {
1425 connection: RedisConnection,
1426 attempts: usize,
1427 elapsed: Duration,
1428}
1429
1430enum CancellableReconnect {
1431 Connected {
1432 connection: RedisConnection,
1433 attempts: usize,
1434 elapsed: Duration,
1435 },
1436 Failed {
1437 attempts: usize,
1438 last_error: RedisError,
1439 },
1440 Closed,
1441}
1442
1443async fn reconnect_campaign_until_closed(
1448 factory: &dyn ConnectionFactory,
1449 config: &ReconnectConfig,
1450 closed: &mut watch::Receiver<bool>,
1451 lifecycle: Option<&ResilientLifecycleControl>,
1452) -> CancellableReconnect {
1453 let max = config.max_retries.unwrap_or(usize::MAX);
1454 let started = Instant::now();
1455 let mut last_error = None;
1456 for attempt in 0..=max {
1457 if *closed.borrow() {
1458 return CancellableReconnect::Closed;
1459 }
1460
1461 let delay = config.delay_for_attempt(attempt);
1462 tracing::warn!(attempt, delay = ?delay, "redis: backing off before reconnect");
1463 if let Some(lifecycle) = lifecycle
1464 && !lifecycle.publish(ConnectionEvent::ReconnectAttempt {
1465 attempt: attempt + 1,
1466 delay,
1467 })
1468 {
1469 return CancellableReconnect::Closed;
1470 }
1471 if !delay.is_zero() {
1472 tokio::select! {
1473 biased;
1475 result = closed.changed() => {
1476 let _ = result;
1477 return CancellableReconnect::Closed;
1478 }
1479 () = tokio::time::sleep(delay) => {}
1480 }
1481 }
1482
1483 let connect = connect_with_timeout(factory, config.connect_timeout);
1484 tokio::pin!(connect);
1485 let result = tokio::select! {
1486 biased;
1489 result = closed.changed() => {
1490 let _ = result;
1491 return CancellableReconnect::Closed;
1492 }
1493 result = &mut connect => result,
1494 };
1495
1496 match result {
1497 Ok(conn) => {
1498 tracing::info!(attempt, "redis: reconnected successfully");
1499 return CancellableReconnect::Connected {
1500 connection: conn,
1501 attempts: attempt + 1,
1502 elapsed: started.elapsed(),
1503 };
1504 }
1505 Err(error) => {
1506 tracing::warn!(attempt, error = %error, "redis: reconnect attempt failed");
1507 if let Some(lifecycle) = lifecycle
1508 && !lifecycle.publish_with(|| ConnectionEvent::ReconnectFailed {
1509 attempt: attempt + 1,
1510 error: Arc::from(error.to_string()),
1511 })
1512 {
1513 return CancellableReconnect::Closed;
1514 }
1515 last_error = Some(error);
1516 }
1517 }
1518 }
1519
1520 let attempts = max.saturating_add(1);
1521 if let Some(lifecycle) = lifecycle
1522 && !lifecycle.publish(ConnectionEvent::ReconnectExhausted { attempts })
1523 {
1524 return CancellableReconnect::Closed;
1525 }
1526 CancellableReconnect::Failed {
1527 attempts,
1528 last_error: last_error.unwrap_or(RedisError::ConnectionClosed),
1529 }
1530}
1531
1532struct ReconnectGate {
1538 lock: Mutex<()>,
1539 generation: AtomicU64,
1540 reconnecting: std::sync::atomic::AtomicBool,
1541}
1542
1543impl ReconnectGate {
1544 fn new() -> Self {
1545 Self {
1546 lock: Mutex::new(()),
1547 generation: AtomicU64::new(0),
1548 reconnecting: std::sync::atomic::AtomicBool::new(false),
1549 }
1550 }
1551
1552 fn generation(&self) -> u64 {
1554 self.generation.load(Ordering::Acquire)
1555 }
1556
1557 async fn enter(&self) -> tokio::sync::MutexGuard<'_, ()> {
1559 self.lock.lock().await
1560 }
1561
1562 fn mark_reconnected(&self) {
1564 self.generation.fetch_add(1, Ordering::AcqRel);
1565 }
1566
1567 fn is_reconnecting(&self) -> bool {
1568 self.reconnecting.load(Ordering::Acquire)
1569 }
1570
1571 fn mark_reconnecting(&self) -> ActiveReconnect<'_> {
1572 self.reconnecting.store(true, Ordering::Release);
1573 ActiveReconnect(&self.reconnecting)
1574 }
1575}
1576
1577struct ActiveReconnect<'a>(&'a std::sync::atomic::AtomicBool);
1578
1579impl Drop for ActiveReconnect<'_> {
1580 fn drop(&mut self) {
1581 self.0.store(false, Ordering::Release);
1582 }
1583}
1584
1585#[cfg(test)]
1586mod tests {
1587 use super::*;
1588 use bytes::Bytes;
1589 use futures::{SinkExt, StreamExt};
1590 use redis_tower_core::{RedisStream, RespCodec, WithDeadline};
1591 use redis_tower_protocol::helpers::{array, bulk};
1592 use std::future::Future;
1593 use std::pin::Pin;
1594 use std::sync::atomic::AtomicUsize;
1595 use std::time::Instant;
1596 use tokio::io::AsyncReadExt;
1597 use tokio::net::{TcpListener, TcpStream};
1598 use tokio::sync::{Semaphore, oneshot};
1599 use tokio_util::codec::Framed;
1600
1601 type ConnFuture = Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>>;
1602
1603 struct FailingFactory {
1605 calls: Arc<AtomicUsize>,
1606 }
1607
1608 impl ConnectionFactory for FailingFactory {
1609 fn connect(&self) -> ConnFuture {
1610 self.calls.fetch_add(1, Ordering::SeqCst);
1611 Box::pin(async { Err(RedisError::ConnectionClosed) })
1612 }
1613 }
1614
1615 struct HangingFactory {
1617 calls: Arc<AtomicUsize>,
1618 }
1619
1620 impl ConnectionFactory for HangingFactory {
1621 fn connect(&self) -> ConnFuture {
1622 self.calls.fetch_add(1, Ordering::SeqCst);
1623 Box::pin(std::future::pending())
1624 }
1625 }
1626
1627 struct ReadyOnCloseFactory {
1631 calls: Arc<AtomicUsize>,
1632 initial: StdMutex<Option<RedisConnection>>,
1633 ready: Arc<AtomicBool>,
1634 }
1635
1636 impl ConnectionFactory for ReadyOnCloseFactory {
1637 fn connect(&self) -> ConnFuture {
1638 let call = self.calls.fetch_add(1, Ordering::SeqCst);
1639 if call == 0 {
1640 let connection = self
1641 .initial
1642 .lock()
1643 .expect("initial connection mutex poisoned")
1644 .take();
1645 return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1646 }
1647
1648 let ready = Arc::clone(&self.ready);
1649 Box::pin(std::future::poll_fn(move |_cx| {
1650 if ready.load(Ordering::Acquire) {
1651 std::task::Poll::Ready(Err(RedisError::ConnectionClosed))
1652 } else {
1653 std::task::Poll::Pending
1654 }
1655 }))
1656 }
1657 }
1658
1659 struct EchoCommand {
1660 value: Bytes,
1661 idempotent: bool,
1662 }
1663
1664 impl EchoCommand {
1665 fn new(value: &'static str, idempotent: bool) -> Self {
1666 Self {
1667 value: Bytes::from_static(value.as_bytes()),
1668 idempotent,
1669 }
1670 }
1671 }
1672
1673 impl Command for EchoCommand {
1674 type Response = Bytes;
1675
1676 fn to_frame(&self) -> Frame {
1677 array(vec![bulk("ECHO"), bulk(self.value.clone())])
1678 }
1679
1680 fn parse_response(&self, frame: Frame) -> Result<Self::Response, RedisError> {
1681 match frame {
1682 Frame::BulkString(Some(value)) => Ok(value),
1683 other => Err(RedisError::UnexpectedResponse {
1684 expected: "bulk string",
1685 actual: format!("{other:?}"),
1686 }),
1687 }
1688 }
1689
1690 fn name(&self) -> &str {
1691 "ECHO"
1692 }
1693
1694 fn idempotent(&self) -> bool {
1695 self.idempotent
1696 }
1697 }
1698
1699 struct ControlledFactory {
1702 calls: Arc<AtomicUsize>,
1703 initial: StdMutex<Option<RedisConnection>>,
1704 replacement: Arc<StdMutex<Option<RedisConnection>>>,
1705 release_reconnect: Arc<Semaphore>,
1706 }
1707
1708 impl ConnectionFactory for ControlledFactory {
1709 fn connect(&self) -> ConnFuture {
1710 let call = self.calls.fetch_add(1, Ordering::SeqCst);
1711 if call == 0 {
1712 let connection = self
1713 .initial
1714 .lock()
1715 .expect("initial connection mutex poisoned")
1716 .take();
1717 return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1718 }
1719
1720 let replacement = Arc::clone(&self.replacement);
1721 let release = Arc::clone(&self.release_reconnect);
1722 Box::pin(async move {
1723 let permit = release
1724 .acquire_owned()
1725 .await
1726 .map_err(|_| RedisError::ConnectionClosed)?;
1727 permit.forget();
1728 replacement
1729 .lock()
1730 .expect("replacement connection mutex poisoned")
1731 .take()
1732 .ok_or(RedisError::ConnectionClosed)
1733 })
1734 }
1735 }
1736
1737 struct SequencedFactory {
1741 calls: Arc<AtomicUsize>,
1742 initial: StdMutex<Option<RedisConnection>>,
1743 replacements: Arc<StdMutex<VecDeque<RedisConnection>>>,
1744 release_reconnect: Arc<Semaphore>,
1745 }
1746
1747 impl ConnectionFactory for SequencedFactory {
1748 fn connect(&self) -> ConnFuture {
1749 let call = self.calls.fetch_add(1, Ordering::SeqCst);
1750 if call == 0 {
1751 let connection = self
1752 .initial
1753 .lock()
1754 .expect("initial connection mutex poisoned")
1755 .take();
1756 return Box::pin(async move { connection.ok_or(RedisError::ConnectionClosed) });
1757 }
1758
1759 let replacements = Arc::clone(&self.replacements);
1760 let release = Arc::clone(&self.release_reconnect);
1761 Box::pin(async move {
1762 let permit = release
1763 .acquire_owned()
1764 .await
1765 .map_err(|_| RedisError::ConnectionClosed)?;
1766 permit.forget();
1767 replacements
1768 .lock()
1769 .expect("replacement connections mutex poisoned")
1770 .pop_front()
1771 .ok_or(RedisError::ConnectionClosed)
1772 })
1773 }
1774 }
1775
1776 async fn connection_pair() -> (RedisConnection, TcpStream) {
1777 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1778 let addr = listener.local_addr().unwrap();
1779 let (client, accepted) = tokio::join!(TcpStream::connect(addr), listener.accept());
1780 let client = client.unwrap();
1781 let (server, _) = accepted.unwrap();
1782 (
1783 RedisConnection::from_stream(RedisStream::Tcp(client)),
1784 server,
1785 )
1786 }
1787
1788 fn close_on_first_request(mut server: TcpStream) -> oneshot::Receiver<()> {
1789 let (seen_tx, seen_rx) = oneshot::channel();
1790 tokio::spawn(async move {
1791 let mut bytes = [0; 1024];
1792 let read = server.read(&mut bytes).await.unwrap();
1793 assert!(read > 0, "mock Redis did not receive the first command");
1794 let _ = seen_tx.send(());
1795 });
1798 seen_rx
1799 }
1800
1801 fn hold_after_first_request(
1802 mut server: TcpStream,
1803 ) -> (oneshot::Receiver<()>, tokio::task::JoinHandle<()>) {
1804 let (seen_tx, seen_rx) = oneshot::channel();
1805 let task = tokio::spawn(async move {
1806 let mut bytes = [0; 1024];
1807 let read = server.read(&mut bytes).await.unwrap();
1808 assert!(read > 0, "mock Redis did not receive the held command");
1809 let _ = seen_tx.send(());
1810 std::future::pending::<()>().await;
1811 });
1812 (seen_rx, task)
1813 }
1814
1815 fn delayed_response_after_first_request(
1816 server: TcpStream,
1817 response: Bytes,
1818 ) -> (
1819 oneshot::Receiver<()>,
1820 oneshot::Sender<()>,
1821 tokio::task::JoinHandle<()>,
1822 ) {
1823 let (seen_tx, seen_rx) = oneshot::channel();
1824 let (respond_tx, respond_rx) = oneshot::channel();
1825 let task = tokio::spawn(async move {
1826 let mut framed = Framed::new(server, RespCodec::new());
1827 framed
1828 .next()
1829 .await
1830 .expect("mock Redis connection closed before held request")
1831 .expect("mock Redis received invalid RESP");
1832 let _ = seen_tx.send(());
1833 respond_rx.await.expect("late response signal dropped");
1834 framed
1835 .send(Frame::BulkString(Some(response)))
1836 .await
1837 .expect("late response should reach the quarantined socket");
1838 });
1839 (seen_rx, respond_tx, task)
1840 }
1841
1842 fn echo_server(server: TcpStream, expected: usize) -> tokio::task::JoinHandle<Vec<Bytes>> {
1843 tokio::spawn(async move {
1844 let mut framed = Framed::new(server, RespCodec::new());
1845 let mut seen = Vec::with_capacity(expected);
1846 while seen.len() < expected {
1847 let request = framed
1848 .next()
1849 .await
1850 .expect("mock Redis connection closed early")
1851 .expect("mock Redis received invalid RESP");
1852 let value = match request {
1853 Frame::Array(Some(parts)) => match parts.get(1) {
1854 Some(Frame::BulkString(Some(value))) => value.clone(),
1855 other => panic!("expected ECHO bulk argument, got {other:?}"),
1856 },
1857 other => panic!("expected command array, got {other:?}"),
1858 };
1859 seen.push(value.clone());
1860 framed.send(Frame::BulkString(Some(value))).await.unwrap();
1861 }
1862 seen
1863 })
1864 }
1865
1866 async fn wait_for_depth(client: &ResilientRedisClient, expected: usize) {
1867 tokio::time::timeout(Duration::from_secs(1), async {
1868 loop {
1869 if client.offline_queue_depth() == expected {
1870 return;
1871 }
1872 tokio::task::yield_now().await;
1873 }
1874 })
1875 .await
1876 .unwrap_or_else(|_| panic!("offline queue never reached depth {expected}"));
1877 }
1878
1879 async fn wait_for_calls(calls: &AtomicUsize, expected: usize) {
1880 tokio::time::timeout(Duration::from_secs(1), async {
1881 loop {
1882 if calls.load(Ordering::SeqCst) == expected {
1883 return;
1884 }
1885 tokio::task::yield_now().await;
1886 }
1887 })
1888 .await
1889 .unwrap_or_else(|_| panic!("factory never reached {expected} calls"));
1890 }
1891
1892 fn config(
1893 max_retries: usize,
1894 base: Duration,
1895 connect_timeout: Option<Duration>,
1896 ) -> ReconnectConfig {
1897 ReconnectConfig {
1898 max_retries: Some(max_retries),
1899 base_delay: base,
1900 jitter: false,
1901 connect_timeout,
1902 ..Default::default()
1903 }
1904 }
1905
1906 #[tokio::test]
1909 async fn connect_with_timeout_propagates_factory_error() {
1910 let calls = Arc::new(AtomicUsize::new(0));
1911 let factory = FailingFactory {
1912 calls: calls.clone(),
1913 };
1914 let result = connect_with_timeout(&factory, None).await;
1916 assert!(matches!(result, Err(RedisError::ConnectionClosed)));
1917 assert_eq!(calls.load(Ordering::SeqCst), 1);
1918 }
1919
1920 #[tokio::test]
1921 async fn connect_with_timeout_times_out_a_hanging_connect() {
1922 let calls = Arc::new(AtomicUsize::new(0));
1923 let factory = HangingFactory {
1924 calls: calls.clone(),
1925 };
1926 let start = Instant::now();
1927 let result = connect_with_timeout(&factory, Some(Duration::from_millis(30))).await;
1928 assert!(matches!(result, Err(RedisError::ConnectTimeout)));
1930 assert!(start.elapsed() < Duration::from_secs(2));
1931 assert_eq!(calls.load(Ordering::SeqCst), 1);
1932 }
1933
1934 #[tokio::test]
1937 async fn reconnect_campaign_applies_base_delay_before_first_attempt() {
1938 let calls = Arc::new(AtomicUsize::new(0));
1939 let factory = FailingFactory {
1940 calls: calls.clone(),
1941 };
1942 let initial_delay = Duration::from_millis(20);
1943 let cfg = config(0, initial_delay, None);
1944 let events = ConnectionEventBus::new(2);
1945 let mut stream = events.subscribe();
1946 let lifecycle = ResilientLifecycleControl::new(events.clone());
1947 let mut campaign = Box::pin(reconnect_campaign(&factory, &cfg, Some(&lifecycle)));
1948
1949 assert!(futures::poll!(campaign.as_mut()).is_pending());
1950 assert_eq!(calls.load(Ordering::SeqCst), 0);
1951 assert_eq!(
1952 stream.recv().await.unwrap(),
1953 ConnectionEvent::ReconnectAttempt {
1954 attempt: 1,
1955 delay: initial_delay,
1956 }
1957 );
1958
1959 let result = tokio::time::timeout(Duration::from_secs(1), campaign)
1960 .await
1961 .expect("campaign should run after its initial delay");
1962 assert!(result.is_none());
1963 assert_eq!(calls.load(Ordering::SeqCst), 1);
1964 }
1965
1966 #[tokio::test]
1967 async fn reconnect_campaign_backs_off_between_retries() {
1968 let calls = Arc::new(AtomicUsize::new(0));
1969 let factory = FailingFactory {
1970 calls: calls.clone(),
1971 };
1972 let cfg = config(2, Duration::from_millis(1), None);
1974 let result = reconnect_campaign(&factory, &cfg, None).await;
1975 assert!(result.is_none());
1976 assert_eq!(calls.load(Ordering::SeqCst), 3);
1977 }
1978
1979 #[tokio::test]
1980 async fn reconnect_campaign_honors_connect_timeout_per_attempt() {
1981 let calls = Arc::new(AtomicUsize::new(0));
1982 let factory = HangingFactory {
1983 calls: calls.clone(),
1984 };
1985 let cfg = config(0, Duration::from_millis(1), Some(Duration::from_millis(30)));
1986 let result = reconnect_campaign(&factory, &cfg, None).await;
1987 assert!(result.is_none());
1988 assert_eq!(calls.load(Ordering::SeqCst), 1);
1989 }
1990
1991 #[tokio::test]
1992 async fn reconnect_campaign_publishes_failure_and_exhaustion_in_order() {
1993 let factory = FailingFactory {
1994 calls: Arc::new(AtomicUsize::new(0)),
1995 };
1996 let events = ConnectionEventBus::new(8);
1997 let mut stream = events.subscribe();
1998 let lifecycle = ResilientLifecycleControl::new(events.clone());
1999 let cfg = config(1, Duration::ZERO, None);
2000 assert!(
2001 reconnect_campaign(&factory, &cfg, Some(&lifecycle))
2002 .await
2003 .is_none()
2004 );
2005
2006 for attempt in 1..=2 {
2007 assert_eq!(
2008 stream.recv().await.unwrap(),
2009 ConnectionEvent::ReconnectAttempt {
2010 attempt,
2011 delay: Duration::ZERO,
2012 }
2013 );
2014 assert!(matches!(
2015 stream.recv().await.unwrap(),
2016 ConnectionEvent::ReconnectFailed {
2017 attempt: failed,
2018 ..
2019 } if failed == attempt
2020 ));
2021 }
2022 assert_eq!(
2023 stream.recv().await.unwrap(),
2024 ConnectionEvent::ReconnectExhausted { attempts: 2 }
2025 );
2026 }
2027
2028 #[tokio::test]
2029 async fn event_constructor_publishes_initial_connect_failure() {
2030 let events = ConnectionEventBus::new(2);
2031 let mut stream = events.subscribe();
2032 let result = ResilientRedisClient::with_config_and_events(
2033 FailingFactory {
2034 calls: Arc::new(AtomicUsize::new(0)),
2035 },
2036 config(0, Duration::ZERO, None),
2037 events,
2038 )
2039 .await;
2040 assert!(matches!(result, Err(RedisError::ConnectionClosed)));
2041 assert!(matches!(
2042 stream.recv().await.unwrap(),
2043 ConnectionEvent::ConnectFailed { .. }
2044 ));
2045 }
2046
2047 #[tokio::test]
2048 async fn event_clients_publish_shutdown_once_when_final_clone_drops() {
2049 use crate::reconnect::ConnectionDisconnectReason;
2050
2051 for offline_queue in [false, true] {
2052 let (connection, _server) = connection_pair().await;
2053 let initial = Arc::new(StdMutex::new(Some(connection)));
2054 let factory = {
2055 let initial = Arc::clone(&initial);
2056 move || {
2057 let connection = initial
2058 .lock()
2059 .expect("initial connection mutex poisoned")
2060 .take();
2061 async move { connection.ok_or(RedisError::ConnectionClosed) }
2062 }
2063 };
2064 let events = ConnectionEventBus::new(4);
2065 let mut stream = events.subscribe();
2066 let client = if offline_queue {
2067 ResilientRedisClient::with_config_and_offline_queue_and_events(
2068 factory,
2069 config(0, Duration::ZERO, None),
2070 OfflineQueueConfig::new(1),
2071 events.clone(),
2072 )
2073 .await
2074 .unwrap()
2075 } else {
2076 ResilientRedisClient::with_config_and_events(
2077 factory,
2078 config(0, Duration::ZERO, None),
2079 events.clone(),
2080 )
2081 .await
2082 .unwrap()
2083 };
2084 assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2085
2086 let last = client.clone();
2087 drop(client);
2088 assert!(
2089 tokio::time::timeout(Duration::from_millis(10), stream.recv())
2090 .await
2091 .is_err(),
2092 "dropping a non-final clone must not emit shutdown"
2093 );
2094 drop(last);
2095 assert_eq!(
2096 stream.recv().await.unwrap(),
2097 ConnectionEvent::Disconnected {
2098 reason: ConnectionDisconnectReason::Shutdown,
2099 }
2100 );
2101 assert!(
2102 tokio::time::timeout(Duration::from_millis(10), stream.recv())
2103 .await
2104 .is_err(),
2105 "final clone drop must emit shutdown exactly once"
2106 );
2107 }
2108 }
2109
2110 #[tokio::test]
2111 async fn lifecycle_gate_orders_in_progress_publication_before_shutdown() {
2112 let events = ConnectionEventBus::new(4);
2113 let mut stream = events.subscribe();
2114 let lease = ResilientClientLifecycle::new(events);
2115 let control = lease.control();
2116 let (entered_tx, entered_rx) = std::sync::mpsc::channel();
2117 let (release_tx, release_rx) = std::sync::mpsc::channel();
2118
2119 let publisher = std::thread::spawn(move || {
2120 control.publish_with(|| {
2121 entered_tx.send(()).unwrap();
2122 release_rx.recv().unwrap();
2123 ConnectionEvent::ReconnectFailed {
2124 attempt: 1,
2125 error: Arc::from("coordinated failure"),
2126 }
2127 })
2128 });
2129 entered_rx
2130 .recv_timeout(Duration::from_secs(1))
2131 .expect("publication never reached the between-check hook");
2132
2133 let (drop_started_tx, drop_started_rx) = std::sync::mpsc::channel();
2134 let dropper = std::thread::spawn(move || {
2135 drop_started_tx.send(()).unwrap();
2136 drop(lease);
2137 });
2138 drop_started_rx
2139 .recv_timeout(Duration::from_secs(1))
2140 .unwrap();
2141 release_tx.send(()).unwrap();
2142
2143 assert!(publisher.join().unwrap());
2144 dropper.join().unwrap();
2145 assert_eq!(
2146 stream.recv().await.unwrap(),
2147 ConnectionEvent::ReconnectFailed {
2148 attempt: 1,
2149 error: Arc::from("coordinated failure"),
2150 }
2151 );
2152 assert_eq!(
2153 stream.recv().await.unwrap(),
2154 ConnectionEvent::Disconnected {
2155 reason: ConnectionDisconnectReason::Shutdown,
2156 }
2157 );
2158 }
2159
2160 #[tokio::test]
2161 async fn final_clone_drop_publishes_shutdown_during_reported_outage() {
2162 let (initial, initial_server) = connection_pair().await;
2163 let initial_seen = close_on_first_request(initial_server);
2164 let (unused_replacement, _replacement_server) = connection_pair().await;
2165 let calls = Arc::new(AtomicUsize::new(0));
2166 let release = Arc::new(Semaphore::new(0));
2167 let factory = ControlledFactory {
2168 calls: Arc::clone(&calls),
2169 initial: StdMutex::new(Some(initial)),
2170 replacement: Arc::new(StdMutex::new(Some(unused_replacement))),
2171 release_reconnect: release,
2172 };
2173 let events = ConnectionEventBus::new(8);
2174 let mut stream = events.subscribe();
2175 let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2176 factory,
2177 config(0, Duration::ZERO, None),
2178 OfflineQueueConfig::new(1),
2179 events.clone(),
2180 )
2181 .await
2182 .unwrap();
2183 assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2184
2185 let command_client = client.clone();
2186 let command = tokio::spawn(async move {
2187 command_client
2188 .execute(EchoCommand::new("outage", true))
2189 .await
2190 });
2191 initial_seen.await.unwrap();
2192 assert!(matches!(
2193 stream.recv().await.unwrap(),
2194 ConnectionEvent::Disconnected {
2195 reason: ConnectionDisconnectReason::ConnectionError { .. }
2196 }
2197 ));
2198 assert_eq!(
2199 stream.recv().await.unwrap(),
2200 ConnectionEvent::ReconnectAttempt {
2201 attempt: 1,
2202 delay: Duration::ZERO,
2203 }
2204 );
2205 wait_for_calls(&calls, 2).await;
2206 assert!(client.is_reconnecting());
2207
2208 command.abort();
2209 assert!(command.await.unwrap_err().is_cancelled());
2210 drop(client);
2211 assert_eq!(
2212 stream.recv().await.unwrap(),
2213 ConnectionEvent::Disconnected {
2214 reason: ConnectionDisconnectReason::Shutdown,
2215 },
2216 "shutdown remains observable after the outage disconnect"
2217 );
2218 assert!(
2219 tokio::time::timeout(Duration::from_millis(10), stream.recv())
2220 .await
2221 .is_err(),
2222 "terminal shutdown must be published exactly once"
2223 );
2224 }
2225
2226 #[tokio::test]
2227 async fn shutdown_is_terminal_when_factory_and_queue_close_are_ready_together() {
2228 let (initial, initial_server) = connection_pair().await;
2229 let initial_seen = close_on_first_request(initial_server);
2230 let calls = Arc::new(AtomicUsize::new(0));
2231 let ready = Arc::new(AtomicBool::new(false));
2232 let factory = ReadyOnCloseFactory {
2233 calls: Arc::clone(&calls),
2234 initial: StdMutex::new(Some(initial)),
2235 ready: Arc::clone(&ready),
2236 };
2237 let events = ConnectionEventBus::new(8);
2238 let mut stream = events.subscribe();
2239 let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2240 factory,
2241 config(0, Duration::ZERO, None),
2242 OfflineQueueConfig::new(1),
2243 events.clone(),
2244 )
2245 .await
2246 .unwrap();
2247 assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
2248
2249 let command_client = client.clone();
2250 let command = tokio::spawn(async move {
2251 command_client
2252 .execute(EchoCommand::new("simultaneous-close", true))
2253 .await
2254 });
2255 initial_seen.await.unwrap();
2256 assert!(matches!(
2257 stream.recv().await.unwrap(),
2258 ConnectionEvent::Disconnected {
2259 reason: ConnectionDisconnectReason::ConnectionError { .. }
2260 }
2261 ));
2262 assert_eq!(
2263 stream.recv().await.unwrap(),
2264 ConnectionEvent::ReconnectAttempt {
2265 attempt: 1,
2266 delay: Duration::ZERO,
2267 }
2268 );
2269 wait_for_calls(&calls, 2).await;
2270
2271 drop(client);
2275 ready.store(true, Ordering::Release);
2276 command.abort();
2277 assert!(command.await.unwrap_err().is_cancelled());
2278 assert_eq!(
2279 stream.recv().await.unwrap(),
2280 ConnectionEvent::Disconnected {
2281 reason: ConnectionDisconnectReason::Shutdown,
2282 }
2283 );
2284
2285 for _ in 0..4 {
2286 tokio::task::yield_now().await;
2287 }
2288 assert!(
2289 tokio::time::timeout(Duration::from_millis(10), stream.recv())
2290 .await
2291 .is_err(),
2292 "no reconnect event may be published after terminal shutdown"
2293 );
2294 }
2295
2296 #[tokio::test]
2299 async fn gate_skips_when_another_clone_already_reconnected() {
2300 let gate = ReconnectGate::new();
2301 let seen_a = gate.generation();
2303 let seen_b = gate.generation();
2304
2305 {
2307 let _g = gate.enter().await;
2308 assert_eq!(gate.generation(), seen_a);
2309 gate.mark_reconnected();
2310 }
2311
2312 {
2314 let _g = gate.enter().await;
2315 assert_ne!(gate.generation(), seen_b);
2316 }
2317 }
2318
2319 #[tokio::test]
2320 async fn gate_serializes_concurrent_holders() {
2321 let gate = Arc::new(ReconnectGate::new());
2322 let held = Arc::new(AtomicUsize::new(0));
2323 let max_seen = Arc::new(AtomicUsize::new(0));
2324
2325 let mut handles = Vec::new();
2326 for _ in 0..8 {
2327 let gate = gate.clone();
2328 let held = held.clone();
2329 let max_seen = max_seen.clone();
2330 handles.push(tokio::spawn(async move {
2331 let _g = gate.enter().await;
2332 let now = held.fetch_add(1, Ordering::SeqCst) + 1;
2333 max_seen.fetch_max(now, Ordering::SeqCst);
2334 tokio::task::yield_now().await;
2335 held.fetch_sub(1, Ordering::SeqCst);
2336 }));
2337 }
2338 for h in handles {
2339 h.await.unwrap();
2340 }
2341 assert_eq!(max_seen.load(Ordering::SeqCst), 1);
2343 }
2344
2345 fn wait_decision(decision: QueueDecision) -> (QueuePermit, Option<u64>) {
2346 match decision {
2347 QueueDecision::Wait { permit, campaign } => (permit, campaign),
2348 QueueDecision::Execute => panic!("expected queued decision, got execute"),
2349 QueueDecision::Reject { error, .. } => {
2350 panic!("expected queued decision, got {error:?}")
2351 }
2352 }
2353 }
2354
2355 #[test]
2356 fn offline_queue_config_is_explicit_and_defaults_to_a_bound() {
2357 assert_eq!(OfflineQueueConfig::new(7).capacity(), 7);
2358 assert_eq!(OfflineQueueConfig::new(7).max_replay_attempts(), 3);
2359 assert_eq!(
2360 OfflineQueueConfig::new(7)
2361 .with_max_replay_attempts(5)
2362 .max_replay_attempts(),
2363 5
2364 );
2365 assert_eq!(OfflineQueueConfig::default().capacity(), 1024);
2366
2367 let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(0));
2368 assert!(matches!(
2369 queue.after_connection_error(true),
2370 QueueDecision::Reject {
2371 error: QueueRejection::Full,
2372 campaign: Some(0),
2373 }
2374 ));
2375 assert!(queue.is_reconnecting());
2376 }
2377
2378 #[tokio::test]
2379 async fn offline_queue_is_bounded_ticket_ordered_and_cancellation_safe() {
2380 let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(3));
2381 let (first, campaign) = wait_decision(queue.after_connection_error(true));
2382 assert_eq!(campaign, Some(0));
2383 let (second, campaign) = wait_decision(queue.before_execute(true));
2384 assert_eq!(campaign, None);
2385 let (third, campaign) = wait_decision(queue.before_execute(true));
2386 assert_eq!(campaign, None);
2387 assert_eq!(queue.depth(), 3);
2388
2389 assert!(matches!(
2390 queue.before_execute(true),
2391 QueueDecision::Reject {
2392 error: QueueRejection::Full,
2393 campaign: None,
2394 }
2395 ));
2396 assert!(matches!(
2397 queue.before_execute(false),
2398 QueueDecision::Reject {
2399 error: QueueRejection::ConnectionClosed,
2400 campaign: None,
2401 }
2402 ));
2403
2404 queue.inner.finish_reconnect(0, None);
2405 first.wait_for_turn().await.unwrap();
2406 assert!(
2407 tokio::time::timeout(Duration::from_millis(10), second.wait_for_turn())
2408 .await
2409 .is_err(),
2410 "the second ticket must not overtake the first"
2411 );
2412
2413 drop(first);
2415 second.wait_for_turn().await.unwrap();
2416 drop(second);
2417 third.wait_for_turn().await.unwrap();
2418 drop(third);
2419 assert_eq!(queue.depth(), 0);
2420 assert!(matches!(queue.state().phase, OfflinePhase::Connected));
2421 }
2422
2423 #[tokio::test]
2424 async fn failed_campaign_rejects_every_admitted_ticket_deterministically() {
2425 let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(2));
2426 let (first, campaign) = wait_decision(queue.after_connection_error(true));
2427 let campaign = campaign.unwrap();
2428 let (second, _) = wait_decision(queue.before_execute(true));
2429 let cause = Arc::new(RedisError::ConnectTimeout);
2430 queue.inner.finish_reconnect(
2431 campaign,
2432 Some(ReconnectFailure {
2433 attempts: 3,
2434 last_error: Arc::clone(&cause),
2435 }),
2436 );
2437
2438 let mut waiter_causes = Vec::new();
2439 for permit in [&first, &second] {
2440 let error = permit.wait_for_turn().await.unwrap_err();
2441 let RedisError::ReconnectFailed {
2442 attempts: 3,
2443 last_error,
2444 } = error
2445 else {
2446 panic!("expected structured reconnect exhaustion")
2447 };
2448 assert!(matches!(&*last_error, RedisError::ConnectTimeout));
2449 assert!(Arc::ptr_eq(&last_error, &cause));
2450 waiter_causes.push(last_error);
2451 }
2452 assert!(Arc::ptr_eq(&waiter_causes[0], &waiter_causes[1]));
2453
2454 let QueueDecision::Reject {
2455 error:
2456 QueueRejection::ReconnectFailed {
2457 attempts: 3,
2458 last_error,
2459 },
2460 campaign: None,
2461 } = queue.before_execute(true)
2462 else {
2463 panic!("new work should observe the same reconnect exhaustion")
2464 };
2465 assert!(matches!(&*last_error, RedisError::ConnectTimeout));
2466 assert!(Arc::ptr_eq(&last_error, &cause));
2467
2468 assert!(matches!(
2469 queue.before_execute(false),
2470 QueueDecision::Reject {
2471 error: QueueRejection::ConnectionClosed,
2472 campaign: None,
2473 }
2474 ));
2475
2476 drop(first);
2477 drop(second);
2478 let (replacement, next_campaign) = wait_decision(queue.before_execute(true));
2479 assert_eq!(next_campaign, Some(1));
2480 drop(replacement);
2481 }
2482
2483 #[tokio::test]
2484 async fn closing_queue_wakes_waiters_with_connection_closed() {
2485 let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(1));
2486 let (permit, _) = wait_decision(queue.after_connection_error(true));
2487 queue.inner.close();
2488
2489 assert!(matches!(
2490 permit.wait_for_turn().await,
2491 Err(RedisError::ConnectionClosed)
2492 ));
2493 assert!(matches!(
2494 queue.before_execute(true),
2495 QueueDecision::Reject {
2496 error: QueueRejection::ConnectionClosed,
2497 campaign: None,
2498 }
2499 ));
2500 }
2501
2502 #[tokio::test]
2503 async fn dropping_last_queue_owner_cancels_hanging_reconnect() {
2504 let queue = OfflineQueueHandle::new(OfflineQueueConfig::new(1));
2505 let mut closed = queue.inner.closed_tx.subscribe();
2506 let calls = Arc::new(AtomicUsize::new(0));
2507 let factory = HangingFactory {
2508 calls: Arc::clone(&calls),
2509 };
2510 let config = config(0, Duration::ZERO, None);
2511
2512 let task = tokio::spawn(async move {
2513 reconnect_campaign_until_closed(&factory, &config, &mut closed, None).await
2514 });
2515 while calls.load(Ordering::SeqCst) == 0 {
2516 tokio::task::yield_now().await;
2517 }
2518 drop(queue);
2519
2520 assert!(matches!(
2521 tokio::time::timeout(Duration::from_secs(1), task)
2522 .await
2523 .expect("queue closure should cancel a hanging connect")
2524 .unwrap(),
2525 CancellableReconnect::Closed
2526 ));
2527 }
2528
2529 #[tokio::test]
2530 async fn idempotent_commands_replay_in_ticket_order_with_one_campaign() {
2531 let (initial, initial_server) = connection_pair().await;
2532 let initial_seen = close_on_first_request(initial_server);
2533 let (replacement, replacement_server) = connection_pair().await;
2534 let replacement_task = echo_server(replacement_server, 3);
2535
2536 let calls = Arc::new(AtomicUsize::new(0));
2537 let release = Arc::new(Semaphore::new(0));
2538 let factory = ControlledFactory {
2539 calls: Arc::clone(&calls),
2540 initial: StdMutex::new(Some(initial)),
2541 replacement: Arc::new(StdMutex::new(Some(replacement))),
2542 release_reconnect: Arc::clone(&release),
2543 };
2544 let client = ResilientRedisClient::with_config_and_offline_queue(
2545 factory,
2546 config(0, Duration::ZERO, None),
2547 OfflineQueueConfig::new(3),
2548 )
2549 .await
2550 .unwrap();
2551
2552 let first_client = client.clone();
2553 let first =
2554 tokio::spawn(
2555 async move { first_client.execute(EchoCommand::new("first", true)).await },
2556 );
2557 initial_seen.await.unwrap();
2558 wait_for_depth(&client, 1).await;
2559 assert!(client.is_reconnecting());
2560
2561 let second_client = client.clone();
2562 let second = tokio::spawn(async move {
2563 second_client
2564 .execute(EchoCommand::new("second", true))
2565 .await
2566 });
2567 wait_for_depth(&client, 2).await;
2568 let third_client = client.clone();
2569 let third =
2570 tokio::spawn(
2571 async move { third_client.execute(EchoCommand::new("third", true)).await },
2572 );
2573 wait_for_depth(&client, 3).await;
2574
2575 assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
2576 release.add_permits(1);
2577
2578 assert_eq!(first.await.unwrap().unwrap(), Bytes::from_static(b"first"));
2579 assert_eq!(
2580 second.await.unwrap().unwrap(),
2581 Bytes::from_static(b"second")
2582 );
2583 assert_eq!(third.await.unwrap().unwrap(), Bytes::from_static(b"third"));
2584 assert_eq!(
2585 replacement_task.await.unwrap(),
2586 vec![
2587 Bytes::from_static(b"first"),
2588 Bytes::from_static(b"second"),
2589 Bytes::from_static(b"third"),
2590 ]
2591 );
2592 assert_eq!(client.offline_queue_depth(), 0);
2593 assert!(!client.is_reconnecting());
2594 }
2595
2596 #[tokio::test]
2597 async fn offline_queue_campaign_publishes_ordered_lifecycle_events() {
2598 use crate::reconnect::ConnectionDisconnectReason;
2599
2600 let (initial, initial_server) = connection_pair().await;
2601 let initial_seen = close_on_first_request(initial_server);
2602 let (replacement, replacement_server) = connection_pair().await;
2603 let replacement_task = echo_server(replacement_server, 1);
2604 let release = Arc::new(Semaphore::new(0));
2605 let factory = ControlledFactory {
2606 calls: Arc::new(AtomicUsize::new(0)),
2607 initial: StdMutex::new(Some(initial)),
2608 replacement: Arc::new(StdMutex::new(Some(replacement))),
2609 release_reconnect: Arc::clone(&release),
2610 };
2611 let events = ConnectionEventBus::new(8);
2612 let mut event_stream = events.subscribe();
2613 let client = ResilientRedisClient::with_config_and_offline_queue_and_events(
2614 factory,
2615 config(0, Duration::ZERO, None),
2616 OfflineQueueConfig::new(1),
2617 events,
2618 )
2619 .await
2620 .unwrap();
2621 assert_eq!(
2622 event_stream.recv().await.unwrap(),
2623 ConnectionEvent::Connected
2624 );
2625
2626 let command_client = client.clone();
2627 let command = tokio::spawn(async move {
2628 command_client
2629 .execute(EchoCommand::new("eventful", true))
2630 .await
2631 });
2632 initial_seen.await.unwrap();
2633 assert!(matches!(
2634 event_stream.recv().await.unwrap(),
2635 ConnectionEvent::Disconnected {
2636 reason: ConnectionDisconnectReason::ConnectionError { .. }
2637 }
2638 ));
2639 assert_eq!(
2640 event_stream.recv().await.unwrap(),
2641 ConnectionEvent::ReconnectAttempt {
2642 attempt: 1,
2643 delay: Duration::ZERO,
2644 }
2645 );
2646
2647 release.add_permits(1);
2648 assert_eq!(
2649 command.await.unwrap().unwrap(),
2650 Bytes::from_static(b"eventful")
2651 );
2652 assert!(matches!(
2653 event_stream.recv().await.unwrap(),
2654 ConnectionEvent::Reconnected { attempts: 1, .. }
2655 ));
2656 assert_eq!(
2657 replacement_task.await.unwrap(),
2658 vec![Bytes::from_static(b"eventful")]
2659 );
2660 }
2661
2662 #[tokio::test]
2663 async fn default_client_reconnects_but_does_not_replay_idempotent_command() {
2664 let (initial, initial_server) = connection_pair().await;
2665 let initial_seen = close_on_first_request(initial_server);
2666 let (replacement, mut replacement_server) = connection_pair().await;
2667
2668 let calls = Arc::new(AtomicUsize::new(0));
2669 let release = Arc::new(Semaphore::new(0));
2670 let factory = ControlledFactory {
2671 calls: Arc::clone(&calls),
2672 initial: StdMutex::new(Some(initial)),
2673 replacement: Arc::new(StdMutex::new(Some(replacement))),
2674 release_reconnect: Arc::clone(&release),
2675 };
2676 let client = ResilientRedisClient::with_config(factory, config(0, Duration::ZERO, None))
2677 .await
2678 .unwrap();
2679
2680 let command_client = client.clone();
2681 let command = tokio::spawn(async move {
2682 command_client
2683 .execute(EchoCommand::new("not-replayed", true))
2684 .await
2685 });
2686 initial_seen.await.unwrap();
2687 tokio::time::timeout(Duration::from_secs(1), async {
2688 while calls.load(Ordering::SeqCst) < 2 {
2689 tokio::task::yield_now().await;
2690 }
2691 })
2692 .await
2693 .expect("default reconnect campaign did not start");
2694 assert!(client.is_reconnecting());
2695 release.add_permits(1);
2696
2697 assert!(command.await.unwrap().is_err());
2698 let mut bytes = [0; 128];
2699 assert!(
2700 tokio::time::timeout(
2701 Duration::from_millis(25),
2702 replacement_server.read(&mut bytes)
2703 )
2704 .await
2705 .is_err(),
2706 "default construction must not replay the failed command"
2707 );
2708 assert_eq!(client.offline_queue_depth(), 0);
2709 assert!(!client.is_reconnecting());
2710 }
2711
2712 #[tokio::test]
2713 async fn non_idempotent_commands_fail_instead_of_waiting_or_replaying() {
2714 let (initial, initial_server) = connection_pair().await;
2715 let initial_seen = close_on_first_request(initial_server);
2716 let (replacement, replacement_server) = connection_pair().await;
2717 let replacement_task = echo_server(replacement_server, 1);
2718
2719 let calls = Arc::new(AtomicUsize::new(0));
2720 let release = Arc::new(Semaphore::new(0));
2721 let factory = ControlledFactory {
2722 calls: Arc::clone(&calls),
2723 initial: StdMutex::new(Some(initial)),
2724 replacement: Arc::new(StdMutex::new(Some(replacement))),
2725 release_reconnect: Arc::clone(&release),
2726 };
2727 let client = ResilientRedisClient::with_config_and_offline_queue(
2728 factory,
2729 config(0, Duration::ZERO, None),
2730 OfflineQueueConfig::new(2),
2731 )
2732 .await
2733 .unwrap();
2734
2735 let discovering_client = client.clone();
2736 let discovering = tokio::spawn(async move {
2737 discovering_client
2738 .execute(EchoCommand::new("unsafe-first", false))
2739 .await
2740 });
2741 initial_seen.await.unwrap();
2742 assert!(discovering.await.unwrap().is_err());
2743
2744 tokio::time::timeout(Duration::from_secs(1), async {
2745 while !client.is_reconnecting() {
2746 tokio::task::yield_now().await;
2747 }
2748 })
2749 .await
2750 .expect("reconnect campaign did not start");
2751
2752 let error = client
2753 .execute(EchoCommand::new("unsafe-while-offline", false))
2754 .await
2755 .unwrap_err();
2756 assert!(matches!(error, RedisError::ConnectionClosed));
2757 let mut raw_client = client.clone();
2758 let raw_error = raw_client
2759 .call(array(vec![bulk("ECHO"), bulk("raw-while-offline")]))
2760 .await
2761 .unwrap_err();
2762 assert!(matches!(raw_error, RedisError::ConnectionClosed));
2763 assert_eq!(client.offline_queue_depth(), 0);
2764
2765 let safe_client = client.clone();
2766 let safe =
2767 tokio::spawn(async move { safe_client.execute(EchoCommand::new("safe", true)).await });
2768 wait_for_depth(&client, 1).await;
2769 release.add_permits(1);
2770 assert_eq!(safe.await.unwrap().unwrap(), Bytes::from_static(b"safe"));
2771 assert_eq!(
2772 replacement_task.await.unwrap(),
2773 vec![Bytes::from_static(b"safe")],
2774 "neither non-idempotent command may be replayed"
2775 );
2776 assert_eq!(calls.load(Ordering::SeqCst), 2);
2777 }
2778
2779 #[tokio::test]
2780 async fn already_expired_offline_command_never_reaches_wire() {
2781 let (connection, mut server) = connection_pair().await;
2782 let initial = Arc::new(StdMutex::new(Some(connection)));
2783 let factory = {
2784 let initial = Arc::clone(&initial);
2785 move || {
2786 let connection = initial
2787 .lock()
2788 .expect("initial connection mutex poisoned")
2789 .take();
2790 async move { connection.ok_or(RedisError::ConnectionClosed) }
2791 }
2792 };
2793 let client = ResilientRedisClient::with_config_and_offline_queue(
2794 factory,
2795 config(0, Duration::ZERO, None),
2796 OfflineQueueConfig::new(1),
2797 )
2798 .await
2799 .unwrap();
2800
2801 let result = client
2802 .execute(WithDeadline::new(
2803 EchoCommand::new("expired", true),
2804 tokio::time::Instant::now() - Duration::from_millis(1),
2805 ))
2806 .await;
2807
2808 assert!(matches!(result, Err(RedisError::CommandTimeout)));
2809 assert_eq!(client.offline_queue_depth(), 0);
2810 assert!(!client.is_reconnecting());
2811 let mut bytes = [0; 128];
2812 assert!(
2813 tokio::time::timeout(Duration::from_millis(25), server.read(&mut bytes))
2814 .await
2815 .is_err(),
2816 "an already-expired command reached the socket"
2817 );
2818 }
2819
2820 #[tokio::test]
2821 async fn queued_deadline_expiry_releases_ticket_without_replay() {
2822 let (initial, initial_server) = connection_pair().await;
2823 let initial_seen = close_on_first_request(initial_server);
2824 let (replacement, replacement_server) = connection_pair().await;
2825 let replacement_task = echo_server(replacement_server, 1);
2826
2827 let calls = Arc::new(AtomicUsize::new(0));
2828 let release = Arc::new(Semaphore::new(0));
2829 let factory = ControlledFactory {
2830 calls: Arc::clone(&calls),
2831 initial: StdMutex::new(Some(initial)),
2832 replacement: Arc::new(StdMutex::new(Some(replacement))),
2833 release_reconnect: Arc::clone(&release),
2834 };
2835 let client = ResilientRedisClient::with_config_and_offline_queue(
2836 factory,
2837 config(0, Duration::ZERO, None),
2838 OfflineQueueConfig::new(2),
2839 )
2840 .await
2841 .unwrap();
2842
2843 let expiring_client = client.clone();
2844 let expiring = tokio::spawn(async move {
2845 expiring_client
2846 .execute(WithDeadline::after(
2847 EchoCommand::new("expired", true),
2848 Duration::from_millis(250),
2849 ))
2850 .await
2851 });
2852 initial_seen.await.unwrap();
2853 wait_for_depth(&client, 1).await;
2854
2855 assert!(matches!(
2856 expiring.await.unwrap(),
2857 Err(RedisError::CommandTimeout)
2858 ));
2859 wait_for_depth(&client, 0).await;
2860
2861 wait_for_calls(&calls, 2).await;
2864 release.add_permits(1);
2865 assert_eq!(
2866 client
2867 .execute(EchoCommand::new("later", true))
2868 .await
2869 .unwrap(),
2870 Bytes::from_static(b"later")
2871 );
2872 assert_eq!(
2873 replacement_task.await.unwrap(),
2874 vec![Bytes::from_static(b"later")]
2875 );
2876 assert_eq!(client.offline_queue_depth(), 0);
2877 }
2878
2879 #[tokio::test]
2880 async fn in_wire_deadline_timeout_quarantines_socket_before_successor() {
2881 let (initial, initial_server) = connection_pair().await;
2882 let (initial_seen, held_server) = hold_after_first_request(initial_server);
2883 let (replacement, replacement_server) = connection_pair().await;
2884 let replacement_task = echo_server(replacement_server, 1);
2885
2886 let calls = Arc::new(AtomicUsize::new(0));
2887 let release = Arc::new(Semaphore::new(0));
2888 let factory = ControlledFactory {
2889 calls: Arc::clone(&calls),
2890 initial: StdMutex::new(Some(initial)),
2891 replacement: Arc::new(StdMutex::new(Some(replacement))),
2892 release_reconnect: Arc::clone(&release),
2893 };
2894 let client = ResilientRedisClient::with_config_and_offline_queue(
2895 factory,
2896 config(0, Duration::ZERO, None),
2897 OfflineQueueConfig::new(2),
2898 )
2899 .await
2900 .unwrap();
2901
2902 let timed_client = client.clone();
2903 let timed = tokio::spawn(async move {
2904 timed_client
2905 .execute(WithDeadline::after(
2906 EchoCommand::new("timed-out", true),
2907 Duration::from_millis(250),
2908 ))
2909 .await
2910 });
2911 initial_seen.await.unwrap();
2912 assert!(matches!(
2913 timed.await.unwrap(),
2914 Err(RedisError::CommandTimeout)
2915 ));
2916
2917 wait_for_calls(&calls, 2).await;
2920 assert!(client.is_reconnecting());
2921 let later_client = client.clone();
2922 let later =
2923 tokio::spawn(
2924 async move { later_client.execute(EchoCommand::new("later", true)).await },
2925 );
2926 wait_for_depth(&client, 1).await;
2927 release.add_permits(1);
2928
2929 assert_eq!(later.await.unwrap().unwrap(), Bytes::from_static(b"later"));
2930 assert_eq!(
2931 replacement_task.await.unwrap(),
2932 vec![Bytes::from_static(b"later")]
2933 );
2934 assert_eq!(calls.load(Ordering::SeqCst), 2);
2935 assert_eq!(client.offline_queue_depth(), 0);
2936 assert!(!client.is_reconnecting());
2937 held_server.abort();
2938 }
2939
2940 #[tokio::test]
2941 async fn canceling_head_command_does_not_cancel_shared_reconnect() {
2942 let (initial, initial_server) = connection_pair().await;
2943 let initial_seen = close_on_first_request(initial_server);
2944 let (replacement, replacement_server) = connection_pair().await;
2945 let replacement_task = echo_server(replacement_server, 1);
2946
2947 let calls = Arc::new(AtomicUsize::new(0));
2948 let release = Arc::new(Semaphore::new(0));
2949 let factory = ControlledFactory {
2950 calls: Arc::clone(&calls),
2951 initial: StdMutex::new(Some(initial)),
2952 replacement: Arc::new(StdMutex::new(Some(replacement))),
2953 release_reconnect: Arc::clone(&release),
2954 };
2955 let client = ResilientRedisClient::with_config_and_offline_queue(
2956 factory,
2957 config(0, Duration::ZERO, None),
2958 OfflineQueueConfig::new(2),
2959 )
2960 .await
2961 .unwrap();
2962
2963 let canceled_client = client.clone();
2964 let canceled = tokio::spawn(async move {
2965 canceled_client
2966 .execute(EchoCommand::new("canceled", true))
2967 .await
2968 });
2969 initial_seen.await.unwrap();
2970 wait_for_depth(&client, 1).await;
2971
2972 let surviving_client = client.clone();
2973 let surviving = tokio::spawn(async move {
2974 surviving_client
2975 .execute(EchoCommand::new("survives", true))
2976 .await
2977 });
2978 wait_for_depth(&client, 2).await;
2979 assert!(matches!(
2980 client.execute(EchoCommand::new("overflow", true)).await,
2981 Err(RedisError::QueueFull)
2982 ));
2983 canceled.abort();
2984 assert!(canceled.await.unwrap_err().is_cancelled());
2985 wait_for_depth(&client, 1).await;
2986
2987 release.add_permits(1);
2988 assert_eq!(
2989 surviving.await.unwrap().unwrap(),
2990 Bytes::from_static(b"survives")
2991 );
2992 assert_eq!(
2993 replacement_task.await.unwrap(),
2994 vec![Bytes::from_static(b"survives")]
2995 );
2996 assert_eq!(calls.load(Ordering::SeqCst), 2);
2997 }
2998
2999 #[tokio::test]
3000 async fn canceling_initial_wire_attempt_replaces_socket_before_later_command() {
3001 let (initial, initial_server) = connection_pair().await;
3002 let (initial_seen, held_server) = hold_after_first_request(initial_server);
3003 let (replacement, replacement_server) = connection_pair().await;
3004 let replacement_task = echo_server(replacement_server, 1);
3005
3006 let calls = Arc::new(AtomicUsize::new(0));
3007 let release = Arc::new(Semaphore::new(0));
3008 let factory = ControlledFactory {
3009 calls: Arc::clone(&calls),
3010 initial: StdMutex::new(Some(initial)),
3011 replacement: Arc::new(StdMutex::new(Some(replacement))),
3012 release_reconnect: Arc::clone(&release),
3013 };
3014 let client = ResilientRedisClient::with_config_and_offline_queue(
3015 factory,
3016 config(0, Duration::ZERO, None),
3017 OfflineQueueConfig::new(2),
3018 )
3019 .await
3020 .unwrap();
3021
3022 let canceled_client = client.clone();
3023 let canceled = tokio::spawn(async move {
3024 canceled_client
3025 .execute(EchoCommand::new("abandoned", true))
3026 .await
3027 });
3028 initial_seen.await.unwrap();
3029 canceled.abort();
3030 assert!(canceled.await.unwrap_err().is_cancelled());
3031
3032 wait_for_calls(&calls, 2).await;
3036 assert!(client.is_reconnecting());
3037 let later_client = client.clone();
3038 let later =
3039 tokio::spawn(
3040 async move { later_client.execute(EchoCommand::new("later", true)).await },
3041 );
3042 wait_for_depth(&client, 1).await;
3043 release.add_permits(1);
3044
3045 assert_eq!(later.await.unwrap().unwrap(), Bytes::from_static(b"later"));
3046 assert_eq!(
3047 replacement_task.await.unwrap(),
3048 vec![Bytes::from_static(b"later")]
3049 );
3050 assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
3051 assert_eq!(client.offline_queue_depth(), 0);
3052 assert!(!client.is_reconnecting());
3053 held_server.abort();
3054 }
3055
3056 #[tokio::test]
3057 async fn canceling_raw_wire_attempt_replaces_socket_before_waiting_successor() {
3058 let (initial, initial_server) = connection_pair().await;
3059 let (initial_seen, send_late_response, late_response_task) =
3060 delayed_response_after_first_request(
3061 initial_server,
3062 Bytes::from_static(b"abandoned-raw"),
3063 );
3064 let (replacement, replacement_server) = connection_pair().await;
3065 let replacement_task = echo_server(replacement_server, 1);
3066
3067 let calls = Arc::new(AtomicUsize::new(0));
3068 let release = Arc::new(Semaphore::new(0));
3069 let factory = ControlledFactory {
3070 calls: Arc::clone(&calls),
3071 initial: StdMutex::new(Some(initial)),
3072 replacement: Arc::new(StdMutex::new(Some(replacement))),
3073 release_reconnect: Arc::clone(&release),
3074 };
3075 let client = ResilientRedisClient::with_config_and_offline_queue(
3076 factory,
3077 config(0, Duration::ZERO, None),
3078 OfflineQueueConfig::new(2),
3079 )
3080 .await
3081 .unwrap();
3082
3083 let mut raw_client = client.clone();
3084 let canceled = tokio::spawn(async move {
3085 raw_client
3086 .call(array(vec![bulk("ECHO"), bulk("abandoned-raw")]))
3087 .await
3088 });
3089 initial_seen.await.unwrap();
3090
3091 let later_client = client.clone();
3095 let later = tokio::spawn(async move {
3096 later_client
3097 .execute(EchoCommand::new("after-raw", true))
3098 .await
3099 });
3100 tokio::task::yield_now().await;
3101 assert!(
3102 !later.is_finished(),
3103 "successor should wait behind raw request"
3104 );
3105
3106 canceled.abort();
3107 assert!(canceled.await.unwrap_err().is_cancelled());
3108 wait_for_calls(&calls, 2).await;
3109 wait_for_depth(&client, 1).await;
3110
3111 send_late_response.send(()).unwrap();
3115 late_response_task.await.unwrap();
3116 assert!(!later.is_finished());
3117 release.add_permits(1);
3118
3119 assert_eq!(
3120 later.await.unwrap().unwrap(),
3121 Bytes::from_static(b"after-raw")
3122 );
3123 assert_eq!(
3124 replacement_task.await.unwrap(),
3125 vec![Bytes::from_static(b"after-raw")]
3126 );
3127 assert_eq!(calls.load(Ordering::SeqCst), 2, "one reconnect campaign");
3128 assert_eq!(client.offline_queue_depth(), 0);
3129 assert!(!client.is_reconnecting());
3130 }
3131
3132 #[tokio::test]
3133 async fn canceling_queued_wire_replay_replaces_socket_before_next_ticket() {
3134 let (initial, initial_server) = connection_pair().await;
3135 let initial_seen = close_on_first_request(initial_server);
3136 let (first_replacement, first_replacement_server) = connection_pair().await;
3137 let (replay_seen, held_replay_server) = hold_after_first_request(first_replacement_server);
3138 let (fresh_replacement, fresh_replacement_server) = connection_pair().await;
3139 let fresh_server_task = echo_server(fresh_replacement_server, 1);
3140
3141 let calls = Arc::new(AtomicUsize::new(0));
3142 let release = Arc::new(Semaphore::new(0));
3143 let factory = SequencedFactory {
3144 calls: Arc::clone(&calls),
3145 initial: StdMutex::new(Some(initial)),
3146 replacements: Arc::new(StdMutex::new(VecDeque::from([
3147 first_replacement,
3148 fresh_replacement,
3149 ]))),
3150 release_reconnect: Arc::clone(&release),
3151 };
3152 let client = ResilientRedisClient::with_config_and_offline_queue(
3153 factory,
3154 config(0, Duration::ZERO, None),
3155 OfflineQueueConfig::new(2),
3156 )
3157 .await
3158 .unwrap();
3159
3160 let canceled_client = client.clone();
3161 let canceled = tokio::spawn(async move {
3162 canceled_client
3163 .execute(EchoCommand::new("head", true))
3164 .await
3165 });
3166 initial_seen.await.unwrap();
3167 wait_for_depth(&client, 1).await;
3168 wait_for_calls(&calls, 2).await;
3169 release.add_permits(1);
3170 replay_seen.await.unwrap();
3171
3172 canceled.abort();
3173 assert!(canceled.await.unwrap_err().is_cancelled());
3174 wait_for_calls(&calls, 3).await;
3175 assert!(client.is_reconnecting());
3176
3177 let later_client = client.clone();
3178 let later = tokio::spawn(async move {
3179 later_client
3180 .execute(EchoCommand::new("next-ticket", true))
3181 .await
3182 });
3183 wait_for_depth(&client, 1).await;
3184 release.add_permits(1);
3185
3186 assert_eq!(
3187 later.await.unwrap().unwrap(),
3188 Bytes::from_static(b"next-ticket")
3189 );
3190 assert_eq!(
3191 fresh_server_task.await.unwrap(),
3192 vec![Bytes::from_static(b"next-ticket")]
3193 );
3194 assert_eq!(calls.load(Ordering::SeqCst), 3, "two reconnect campaigns");
3195 assert_eq!(client.offline_queue_depth(), 0);
3196 assert!(!client.is_reconnecting());
3197 held_replay_server.abort();
3198 }
3199
3200 #[tokio::test]
3201 async fn replay_attempt_limit_bounds_successful_connect_disconnect_churn() {
3202 let (initial, initial_server) = connection_pair().await;
3203 let initial_seen = close_on_first_request(initial_server);
3204 let (replacement_one, server_one) = connection_pair().await;
3205 let replay_one_seen = close_on_first_request(server_one);
3206 let (replacement_two, server_two) = connection_pair().await;
3207 let replay_two_seen = close_on_first_request(server_two);
3208 let (repair, repair_server) = connection_pair().await;
3209 let repair_task = echo_server(repair_server, 1);
3210
3211 let calls = Arc::new(AtomicUsize::new(0));
3212 let release = Arc::new(Semaphore::new(0));
3213 let factory = SequencedFactory {
3214 calls: Arc::clone(&calls),
3215 initial: StdMutex::new(Some(initial)),
3216 replacements: Arc::new(StdMutex::new(VecDeque::from([
3217 replacement_one,
3218 replacement_two,
3219 repair,
3220 ]))),
3221 release_reconnect: Arc::clone(&release),
3222 };
3223 let client = ResilientRedisClient::with_config_and_offline_queue(
3224 factory,
3225 config(0, Duration::ZERO, None),
3226 OfflineQueueConfig::new(2).with_max_replay_attempts(2),
3227 )
3228 .await
3229 .unwrap();
3230
3231 let head_client = client.clone();
3232 let head = tokio::spawn(async move {
3233 head_client
3234 .execute(EchoCommand::new("bounded-head", true))
3235 .await
3236 });
3237 initial_seen.await.unwrap();
3238 wait_for_calls(&calls, 2).await;
3239 release.add_permits(1);
3240 replay_one_seen.await.unwrap();
3241 wait_for_calls(&calls, 3).await;
3242 release.add_permits(1);
3243 replay_two_seen.await.unwrap();
3244
3245 let error = head.await.unwrap().unwrap_err();
3246 assert!(matches!(
3247 error,
3248 RedisError::ReconnectFailed { attempts: 2, .. }
3249 ));
3250 wait_for_calls(&calls, 4).await;
3251 assert_eq!(client.offline_queue_depth(), 0);
3252 assert!(client.is_reconnecting());
3253
3254 let later_client = client.clone();
3255 let later = tokio::spawn(async move {
3256 later_client
3257 .execute(EchoCommand::new("after-limit", true))
3258 .await
3259 });
3260 wait_for_depth(&client, 1).await;
3261 release.add_permits(1);
3262 assert_eq!(
3263 later.await.unwrap().unwrap(),
3264 Bytes::from_static(b"after-limit")
3265 );
3266 assert_eq!(
3267 repair_task.await.unwrap(),
3268 vec![Bytes::from_static(b"after-limit")]
3269 );
3270 assert_eq!(calls.load(Ordering::SeqCst), 4);
3271 assert_eq!(client.offline_queue_depth(), 0);
3272 assert!(!client.is_reconnecting());
3273 }
3274}