1use std::future::Future;
29use std::pin::Pin;
30use std::sync::{Arc, Mutex};
31use std::task::{Context, Poll};
32use std::time::{Duration, Instant};
33
34use redis_tower_core::{Frame, ReceivedPushFrame, RedisConnection, RedisError};
35use tokio::sync::{Mutex as AsyncMutex, Notify, futures::OwnedNotified, mpsc, oneshot, watch};
36use tokio_util::sync::PollSender;
37use tower_service::Service;
38use tracing::warn;
39
40use crate::maintenance::{
41 MaintenanceControl, MaintenanceListenerHandle, MaintenanceState, ParsedMaintenanceNotification,
42 next_listener_id, parse_maintenance_push, register_connection,
43};
44use crate::metrics_layer::MetricsRecorder;
45use crate::reconnect::{
46 ConnectionDisconnectReason, ConnectionEvent, ConnectionEventBus, ConnectionFactory,
47 ReconnectConfig, connect_with_timeout,
48};
49
50#[derive(Clone)]
52pub struct AutoPipelineConfig {
53 pub max_batch_size: usize,
55 pub batch_window: Duration,
61 pub queue_capacity: usize,
71 pub shed_load_on_full: bool,
81 pub response_timeout: Option<Duration>,
97 pub metrics_recorder: Option<Arc<dyn MetricsRecorder>>,
114 pub reconnect_on_readonly: bool,
127}
128
129impl std::fmt::Debug for AutoPipelineConfig {
130 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131 f.debug_struct("AutoPipelineConfig")
132 .field("max_batch_size", &self.max_batch_size)
133 .field("batch_window", &self.batch_window)
134 .field("queue_capacity", &self.queue_capacity)
135 .field("shed_load_on_full", &self.shed_load_on_full)
136 .field("response_timeout", &self.response_timeout)
137 .field(
138 "metrics_recorder",
139 &self.metrics_recorder.as_ref().map(|_| "<recorder>"),
140 )
141 .field("reconnect_on_readonly", &self.reconnect_on_readonly)
142 .finish()
143 }
144}
145
146impl Default for AutoPipelineConfig {
147 fn default() -> Self {
148 Self {
149 max_batch_size: 100,
150 batch_window: Duration::ZERO,
151 queue_capacity: 1024,
152 shed_load_on_full: false,
153 response_timeout: None,
154 metrics_recorder: None,
155 reconnect_on_readonly: false,
156 }
157 }
158}
159
160#[derive(Debug, Clone, Default)]
168pub struct AutoPipelineReconnectConfig {
169 pub reconnect: ReconnectConfig,
174}
175
176impl AutoPipelineReconnectConfig {
177 pub fn new(reconnect: ReconnectConfig) -> Self {
179 Self { reconnect }
180 }
181}
182
183enum ConnSource {
185 Fixed,
187 Factory {
189 factory: Arc<dyn ConnectionFactory>,
190 reconnect: ReconnectConfig,
191 },
192}
193
194struct MaintenanceWorker {
195 state: MaintenanceState,
196 control_rx: mpsc::UnboundedReceiver<MaintenanceControl>,
197 push_rx: tokio::sync::broadcast::Receiver<ReceivedPushFrame>,
198}
199
200enum PipelineFailure {
205 Connection(RedisError),
206 CommandTimeout,
207 ReadOnly,
208}
209
210impl PipelineFailure {
211 fn event_reason(&self) -> ConnectionDisconnectReason {
212 match self {
213 Self::Connection(error) => ConnectionDisconnectReason::ConnectionError {
214 error: Arc::from(error.to_string()),
215 },
216 Self::CommandTimeout => ConnectionDisconnectReason::CommandTimeout,
217 Self::ReadOnly => ConnectionDisconnectReason::ReadOnly,
218 }
219 }
220}
221
222enum WorkerRequest {
231 Single {
232 frame: Frame,
233 response_tx: oneshot::Sender<Result<Frame, RedisError>>,
234 },
235 Multi {
236 frames: Vec<Frame>,
237 response_tx: oneshot::Sender<Result<Vec<Frame>, RedisError>>,
238 },
239}
240
241impl WorkerRequest {
242 fn frame_count(&self) -> usize {
243 match self {
244 WorkerRequest::Single { .. } => 1,
245 WorkerRequest::Multi { frames, .. } => frames.len(),
246 }
247 }
248
249 fn fail(self, err: RedisError) {
250 match self {
251 WorkerRequest::Single { response_tx, .. } => {
252 let _ = response_tx.send(Err(err));
253 }
254 WorkerRequest::Multi { response_tx, .. } => {
255 let _ = response_tx.send(Err(err));
256 }
257 }
258 }
259
260 fn response_is_closed(&self) -> bool {
266 match self {
267 WorkerRequest::Single { response_tx, .. } => response_tx.is_closed(),
268 WorkerRequest::Multi { response_tx, .. } => response_tx.is_closed(),
269 }
270 }
271}
272
273pub struct AutoPipelineService {
300 tx: mpsc::Sender<WorkerRequest>,
301 poll_tx: PollSender<WorkerRequest>,
305 shed_load: bool,
309 lease: WorkerLease,
312 shutdown_notified: Pin<Box<OwnedNotified>>,
316 connection_health: watch::Receiver<bool>,
320 worker: Arc<WorkerHandle>,
321}
322
323#[derive(Clone)]
336#[must_use = "retain the handle and call shutdown() during orderly application shutdown"]
337pub struct AutoPipelineShutdownHandle {
338 control: Arc<WorkerControl>,
339 worker: Arc<WorkerHandle>,
340}
341
342impl std::fmt::Debug for AutoPipelineShutdownHandle {
343 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344 formatter
345 .debug_struct("AutoPipelineShutdownHandle")
346 .field("shutdown_requested", &self.is_shutdown_requested())
347 .field("worker_finished", &self.worker.is_finished())
348 .finish()
349 }
350}
351
352impl AutoPipelineShutdownHandle {
353 pub async fn shutdown(&self) {
361 self.control.request_shutdown();
362 self.worker.join().await;
363 }
364
365 pub fn is_shutdown_requested(&self) -> bool {
367 self.control.is_shutdown_requested()
368 }
369
370 pub fn is_finished(&self) -> bool {
372 self.worker.is_finished()
373 }
374}
375
376struct WorkerControl {
377 state: Mutex<WorkerControlState>,
378 shutdown_tx: watch::Sender<bool>,
379 shutdown_notify: Arc<Notify>,
380 event_bus: Option<ConnectionEventBus>,
381}
382
383struct WorkerControlState {
384 handles: usize,
385 worker_running: bool,
386 shutdown_requested: bool,
387 shutdown_published: bool,
388}
389
390impl WorkerControl {
391 fn new(shutdown_tx: watch::Sender<bool>, event_bus: Option<ConnectionEventBus>) -> Self {
392 Self {
393 state: Mutex::new(WorkerControlState {
394 handles: 1,
395 worker_running: true,
396 shutdown_requested: false,
397 shutdown_published: false,
398 }),
399 shutdown_tx,
400 shutdown_notify: Arc::new(Notify::new()),
401 event_bus,
402 }
403 }
404
405 fn add_handle(&self) {
406 let mut state = self
407 .state
408 .lock()
409 .unwrap_or_else(std::sync::PoisonError::into_inner);
410 state.handles = state
411 .handles
412 .checked_add(1)
413 .expect("AutoPipelineService handle count overflowed");
414 }
415
416 fn release_handle(&self) {
417 let should_signal = {
418 let mut state = self
419 .state
420 .lock()
421 .unwrap_or_else(std::sync::PoisonError::into_inner);
422 state.handles = state
423 .handles
424 .checked_sub(1)
425 .expect("AutoPipelineService handle count underflowed");
426 let should_signal = state.handles == 0 && !state.shutdown_requested;
427 if state.handles == 0 {
428 state.shutdown_requested = true;
429 if !state.worker_running {
430 self.publish_shutdown_locked(&mut state);
431 }
432 }
433 should_signal
434 };
435 if should_signal {
436 self.shutdown_tx.send_replace(true);
437 self.shutdown_notify.notify_waiters();
438 }
439 }
440
441 fn request_shutdown(&self) {
442 let should_signal = {
443 let mut state = self
444 .state
445 .lock()
446 .unwrap_or_else(std::sync::PoisonError::into_inner);
447 let should_signal = !state.shutdown_requested;
448 state.shutdown_requested = true;
449 if !state.worker_running {
450 self.publish_shutdown_locked(&mut state);
451 }
452 should_signal
453 };
454 if should_signal {
455 self.shutdown_tx.send_replace(true);
456 self.shutdown_notify.notify_waiters();
457 }
458 }
459
460 fn shutdown_notified(&self) -> OwnedNotified {
461 Arc::clone(&self.shutdown_notify).notified_owned()
462 }
463
464 fn is_shutdown_requested(&self) -> bool {
465 self.state
466 .lock()
467 .unwrap_or_else(std::sync::PoisonError::into_inner)
468 .shutdown_requested
469 }
470
471 fn handle_count(&self) -> usize {
472 self.state
473 .lock()
474 .unwrap_or_else(std::sync::PoisonError::into_inner)
475 .handles
476 }
477
478 fn worker_finished(&self) {
479 let mut state = self
480 .state
481 .lock()
482 .unwrap_or_else(std::sync::PoisonError::into_inner);
483 state.worker_running = false;
484 if state.shutdown_requested {
485 self.publish_shutdown_locked(&mut state);
486 }
487 }
488
489 fn publish_shutdown_locked(&self, state: &mut WorkerControlState) {
490 if state.shutdown_published {
491 return;
492 }
493 state.shutdown_published = true;
494 if let Some(events) = &self.event_bus {
495 events.publish(ConnectionEvent::Disconnected {
496 reason: ConnectionDisconnectReason::Shutdown,
497 });
498 }
499 }
500
501 fn with_active_handle<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
505 let state = self
506 .state
507 .lock()
508 .unwrap_or_else(std::sync::PoisonError::into_inner);
509 if state.handles == 0 || state.shutdown_requested {
510 None
511 } else {
512 Some(f())
513 }
514 }
515
516 fn with_admission<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
518 let state = self
519 .state
520 .lock()
521 .unwrap_or_else(std::sync::PoisonError::into_inner);
522 if state.shutdown_requested {
523 None
524 } else {
525 Some(f())
526 }
527 }
528}
529
530struct WorkerRunGuard {
531 control: Arc<WorkerControl>,
532}
533
534impl Drop for WorkerRunGuard {
535 fn drop(&mut self) {
536 self.control.worker_finished();
537 }
538}
539
540struct WorkerLifecycle {
541 control: Arc<WorkerControl>,
542 shutdown: watch::Receiver<bool>,
543 connection_health: ConnectionHealthPublisher,
544 _run_guard: WorkerRunGuard,
545}
546
547struct ConnectionHealthPublisher {
552 tx: watch::Sender<bool>,
553}
554
555impl ConnectionHealthPublisher {
556 fn new(tx: watch::Sender<bool>) -> Self {
557 Self { tx }
558 }
559
560 fn set(&self, healthy: bool) {
561 self.tx.send_if_modified(|current| {
562 let changed = *current != healthy;
563 *current = healthy;
564 changed
565 });
566 }
567}
568
569impl Drop for ConnectionHealthPublisher {
570 fn drop(&mut self) {
571 self.set(false);
572 }
573}
574
575struct WorkerLease {
576 control: Arc<WorkerControl>,
577}
578
579impl Clone for WorkerLease {
580 fn clone(&self) -> Self {
581 self.control.add_handle();
582 Self {
583 control: Arc::clone(&self.control),
584 }
585 }
586}
587
588impl Drop for WorkerLease {
589 fn drop(&mut self) {
590 self.control.release_handle();
591 }
592}
593
594struct WorkerHandle {
601 handle: AsyncMutex<Option<tokio::task::JoinHandle<()>>>,
602}
603
604impl WorkerHandle {
605 fn new(handle: tokio::task::JoinHandle<()>) -> Self {
606 Self {
607 handle: AsyncMutex::new(Some(handle)),
608 }
609 }
610
611 fn is_finished(&self) -> bool {
613 self.handle
614 .try_lock()
615 .map(|handle| handle.as_ref().is_none_or(|handle| handle.is_finished()))
616 .unwrap_or(false)
617 }
618
619 async fn join(&self) {
621 let mut handle = self.handle.lock().await;
622 if let Some(handle) = handle.as_mut() {
623 let _ = handle.await;
624 }
625 handle.take();
626 }
627}
628
629impl Drop for WorkerHandle {
630 fn drop(&mut self) {
631 let unfinished = self
635 .handle
636 .try_lock()
637 .map(|handle| handle.as_ref().is_some_and(|handle| !handle.is_finished()))
638 .unwrap_or(false);
639 if unfinished {
640 warn!(
641 "AutoPipelineService dropped without calling shutdown(); \
642 background worker may still have queued requests in flight"
643 );
644 }
645 }
646}
647
648impl AutoPipelineService {
649 pub fn new(conn: RedisConnection, config: AutoPipelineConfig) -> Self {
659 Self::from_parts(Some(conn), config, ConnSource::Fixed, None, None)
660 }
661
662 pub fn new_with_events(
669 conn: RedisConnection,
670 config: AutoPipelineConfig,
671 events: ConnectionEventBus,
672 ) -> Self {
673 Self::from_parts(Some(conn), config, ConnSource::Fixed, Some(events), None)
674 }
675
676 pub async fn with_factory(
689 factory: impl ConnectionFactory,
690 config: AutoPipelineConfig,
691 reconnect: AutoPipelineReconnectConfig,
692 ) -> Result<Self, RedisError> {
693 Self::with_factory_inner(factory, config, reconnect, None).await
694 }
695
696 pub async fn with_factory_and_events(
703 factory: impl ConnectionFactory,
704 config: AutoPipelineConfig,
705 reconnect: AutoPipelineReconnectConfig,
706 events: ConnectionEventBus,
707 ) -> Result<Self, RedisError> {
708 Self::with_factory_inner(factory, config, reconnect, Some(events)).await
709 }
710
711 pub async fn with_factory_and_maintenance(
717 factory: impl ConnectionFactory,
718 config: AutoPipelineConfig,
719 reconnect: AutoPipelineReconnectConfig,
720 ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
721 Self::with_factory_and_maintenance_inner(factory, config, reconnect, None).await
722 }
723
724 pub async fn with_factory_and_maintenance_and_events(
726 factory: impl ConnectionFactory,
727 config: AutoPipelineConfig,
728 reconnect: AutoPipelineReconnectConfig,
729 events: ConnectionEventBus,
730 ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
731 Self::with_factory_and_maintenance_inner(factory, config, reconnect, Some(events)).await
732 }
733
734 async fn with_factory_and_maintenance_inner(
735 factory: impl ConnectionFactory,
736 config: AutoPipelineConfig,
737 reconnect: AutoPipelineReconnectConfig,
738 event_bus: Option<ConnectionEventBus>,
739 ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
740 let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
741 let (conn, push_rx) =
742 match connect_and_prepare(factory.as_ref(), reconnect.reconnect.connect_timeout, true)
743 .await
744 {
745 Ok((conn, Some(push_rx))) => (conn, push_rx),
746 Ok((_conn, None)) => {
747 unreachable!("maintenance setup always returns a push receiver")
748 }
749 Err(error) => {
750 if let Some(events) = &event_bus {
751 events.publish_with(|| ConnectionEvent::ConnectFailed {
752 error: Arc::from(error.to_string()),
753 });
754 }
755 return Err(error);
756 }
757 };
758 let listener_id = next_listener_id();
759 let (control_tx, control_rx) = mpsc::unbounded_channel();
760 let source = ConnSource::Factory {
761 factory,
762 reconnect: reconnect.reconnect,
763 };
764 let maintenance = MaintenanceWorker {
765 state: MaintenanceState::new(listener_id),
766 control_rx,
767 push_rx,
768 };
769 let service = Self::from_parts(Some(conn), config, source, event_bus, Some(maintenance));
770 let handle = MaintenanceListenerHandle::new(control_tx, listener_id);
771 Ok((service, handle))
772 }
773
774 async fn with_factory_inner(
775 factory: impl ConnectionFactory,
776 config: AutoPipelineConfig,
777 reconnect: AutoPipelineReconnectConfig,
778 event_bus: Option<ConnectionEventBus>,
779 ) -> Result<Self, RedisError> {
780 let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
781 let conn = match connect_with_timeout(factory.as_ref(), reconnect.reconnect.connect_timeout)
782 .await
783 {
784 Ok(conn) => conn,
785 Err(error) => {
786 if let Some(events) = &event_bus {
787 events.publish_with(|| ConnectionEvent::ConnectFailed {
788 error: Arc::from(error.to_string()),
789 });
790 }
791 return Err(error);
792 }
793 };
794 let source = ConnSource::Factory {
795 factory,
796 reconnect: reconnect.reconnect,
797 };
798 Ok(Self::from_parts(
799 Some(conn),
800 config,
801 source,
802 event_bus,
803 None,
804 ))
805 }
806
807 pub fn with_lazy_factory(
827 factory: impl ConnectionFactory,
828 config: AutoPipelineConfig,
829 reconnect: AutoPipelineReconnectConfig,
830 ) -> Self {
831 Self::with_lazy_factory_inner(factory, config, reconnect, None)
832 }
833
834 pub fn with_lazy_factory_and_events(
843 factory: impl ConnectionFactory,
844 config: AutoPipelineConfig,
845 reconnect: AutoPipelineReconnectConfig,
846 events: ConnectionEventBus,
847 ) -> Self {
848 Self::with_lazy_factory_inner(factory, config, reconnect, Some(events))
849 }
850
851 fn with_lazy_factory_inner(
852 factory: impl ConnectionFactory,
853 config: AutoPipelineConfig,
854 reconnect: AutoPipelineReconnectConfig,
855 event_bus: Option<ConnectionEventBus>,
856 ) -> Self {
857 let source = ConnSource::Factory {
858 factory: Arc::new(factory),
859 reconnect: reconnect.reconnect,
860 };
861 Self::from_parts(None, config, source, event_bus, None)
862 }
863
864 fn from_parts(
865 conn: Option<RedisConnection>,
866 config: AutoPipelineConfig,
867 source: ConnSource,
868 event_bus: Option<ConnectionEventBus>,
869 maintenance: Option<MaintenanceWorker>,
870 ) -> Self {
871 let (tx, rx) = mpsc::channel(config.queue_capacity);
872 let poll_tx = PollSender::new(tx.clone());
873 let shed_load = config.shed_load_on_full;
874 let (shutdown_tx, shutdown_rx) = watch::channel(false);
875 let initially_connected = conn.is_some();
876 let (connection_health_tx, connection_health) = watch::channel(initially_connected);
877 let control = Arc::new(WorkerControl::new(shutdown_tx, event_bus.clone()));
878 let lease = WorkerLease {
879 control: Arc::clone(&control),
880 };
881 if initially_connected && let Some(events) = &event_bus {
882 events.publish(ConnectionEvent::Connected);
883 }
884 let lifecycle = WorkerLifecycle {
885 control: Arc::clone(&control),
886 shutdown: shutdown_rx,
887 connection_health: ConnectionHealthPublisher::new(connection_health_tx),
888 _run_guard: WorkerRunGuard {
889 control: Arc::clone(&control),
890 },
891 };
892 let runtime = tokio::runtime::Handle::try_current()
893 .expect("AutoPipelineService must be constructed inside an entered Tokio runtime");
894 let handle = runtime.spawn(pipeline_worker(
895 rx,
896 conn,
897 config,
898 source,
899 event_bus,
900 lifecycle,
901 maintenance,
902 ));
903 Self {
904 tx,
905 poll_tx,
906 shed_load,
907 lease,
908 shutdown_notified: Box::pin(control.shutdown_notified()),
909 connection_health,
910 worker: Arc::new(WorkerHandle::new(handle)),
911 }
912 }
913
914 pub async fn call_pipeline(&mut self, frames: Vec<Frame>) -> Result<Vec<Frame>, RedisError> {
927 if frames.is_empty() {
928 return Ok(Vec::new());
929 }
930 let (resp_tx, resp_rx) = oneshot::channel();
931 let request = WorkerRequest::Multi {
932 frames,
933 response_tx: resp_tx,
934 };
935 if self.shed_load {
936 self.lease
937 .control
938 .with_admission(|| {
939 self.tx.try_send(request).map_err(|e| match e {
940 tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
941 tokio::sync::mpsc::error::TrySendError::Closed(_) => {
942 RedisError::ConnectionClosed
943 }
944 })
945 })
946 .ok_or(RedisError::ConnectionClosed)??;
947 } else {
948 let shutdown = self.lease.control.shutdown_notified();
952 if self.lease.control.is_shutdown_requested() {
953 return Err(RedisError::ConnectionClosed);
954 }
955 let permit = tokio::select! {
956 biased;
957 () = shutdown => return Err(RedisError::ConnectionClosed),
958 permit = self.tx.reserve() => {
959 permit.map_err(|_| RedisError::ConnectionClosed)?
960 }
961 };
962 self.lease
963 .control
964 .with_admission(|| permit.send(request))
965 .ok_or(RedisError::ConnectionClosed)?;
966 }
967 resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
968 }
969
970 pub(crate) fn call_reserved_pipeline(
984 &mut self,
985 frames: Vec<Frame>,
986 ) -> Pin<Box<dyn Future<Output = Result<Vec<Frame>, RedisError>> + Send>> {
987 if frames.is_empty() {
988 self.release_reservation();
991 return Box::pin(async { Ok(Vec::new()) });
992 }
993
994 let (resp_tx, resp_rx) = oneshot::channel();
995 let request = WorkerRequest::Multi {
996 frames,
997 response_tx: resp_tx,
998 };
999
1000 let send_result = if self.shed_load {
1001 self.lease
1002 .control
1003 .with_admission(|| {
1004 self.tx.try_send(request).map_err(|e| match e {
1005 tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
1006 tokio::sync::mpsc::error::TrySendError::Closed(_) => {
1007 RedisError::ConnectionClosed
1008 }
1009 })
1010 })
1011 .unwrap_or(Err(RedisError::ConnectionClosed))
1012 } else {
1013 self.lease
1014 .control
1015 .with_admission(|| {
1016 self.poll_tx
1017 .send_item(request)
1018 .map_err(|_| RedisError::ConnectionClosed)
1019 })
1020 .unwrap_or_else(|| {
1021 let _ = self.poll_tx.abort_send();
1022 Err(RedisError::ConnectionClosed)
1023 })
1024 };
1025
1026 Box::pin(async move {
1027 send_result?;
1028 resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
1029 })
1030 }
1031
1032 pub(crate) fn release_reservation(&mut self) {
1040 let _ = self.poll_tx.abort_send();
1041 }
1042
1043 pub fn shutdown_handle(&self) -> AutoPipelineShutdownHandle {
1050 AutoPipelineShutdownHandle {
1051 control: Arc::clone(&self.lease.control),
1052 worker: Arc::clone(&self.worker),
1053 }
1054 }
1055
1056 pub async fn shutdown(self) {
1078 let control = Arc::clone(&self.lease.control);
1079 let worker = Arc::clone(&self.worker);
1080 drop(self.tx);
1086 drop(self.poll_tx);
1087 drop(self.lease);
1091 drop(self.worker);
1092 if control.handle_count() == 0 {
1096 worker.join().await;
1097 }
1098 }
1099
1100 pub fn is_alive(&self) -> bool {
1108 !self.worker.is_finished()
1109 }
1110
1111 #[doc(hidden)]
1117 pub fn is_connection_healthy(&self) -> bool {
1118 *self.connection_health.borrow()
1119 }
1120
1121 #[doc(hidden)]
1131 pub fn subscribe_connection_health(&self) -> watch::Receiver<bool> {
1132 self.connection_health.clone()
1133 }
1134}
1135
1136async fn pipeline_worker(
1143 mut rx: mpsc::Receiver<WorkerRequest>,
1144 mut conn: Option<RedisConnection>,
1145 config: AutoPipelineConfig,
1146 source: ConnSource,
1147 event_bus: Option<ConnectionEventBus>,
1148 lifecycle: WorkerLifecycle,
1149 maintenance: Option<MaintenanceWorker>,
1150) {
1151 let WorkerLifecycle {
1152 control,
1153 mut shutdown,
1154 connection_health,
1155 _run_guard,
1156 } = lifecycle;
1157 let mut outage_reported = false;
1158 let mut shutting_down = false;
1159 let (mut maintenance_state, mut maintenance_control_rx, mut maintenance_push_rx) =
1160 match maintenance {
1161 Some(maintenance) => (
1162 Some(maintenance.state),
1163 Some(maintenance.control_rx),
1164 Some(maintenance.push_rx),
1165 ),
1166 None => (None, None, None),
1167 };
1168 'worker: loop {
1169 drain_maintenance_control(
1170 &mut maintenance_state,
1171 &mut maintenance_control_rx,
1172 &mut maintenance_push_rx,
1173 );
1174 if maintenance_state
1175 .as_ref()
1176 .and_then(|state| state.pending.as_ref())
1177 .is_some()
1178 {
1179 match wait_for_pending_handoff(
1180 &mut conn,
1181 &mut maintenance_state,
1182 &mut maintenance_control_rx,
1183 &mut maintenance_push_rx,
1184 event_bus.as_ref(),
1185 &mut shutdown,
1186 )
1187 .await
1188 {
1189 PendingHandoffOutcome::Cancelled => continue,
1190 PendingHandoffOutcome::Shutdown => {
1191 shutting_down = true;
1192 rx.close();
1193 disable_maintenance(
1194 &mut maintenance_state,
1195 &mut maintenance_control_rx,
1196 &mut maintenance_push_rx,
1197 );
1198 continue;
1199 }
1200 PendingHandoffOutcome::ConnectionFailed(error) => {
1201 clear_pending_handoff(&mut maintenance_state);
1202 connection_health.set(false);
1203 let failure = PipelineFailure::Connection(error);
1204 publish_worker_outage(
1205 event_bus.as_ref(),
1206 &mut outage_reported,
1207 failure.event_reason(),
1208 &control,
1209 );
1210 let ConnSource::Factory { factory, reconnect } = &source else {
1211 fail_queued_requests(&mut rx);
1212 return;
1213 };
1214 let maintenance_enabled = maintenance_state
1215 .as_ref()
1216 .is_some_and(|state| state.enabled);
1217 if !reconnect_worker_connection(
1218 conn.as_mut()
1219 .expect("maintenance listener requires a connection"),
1220 &mut rx,
1221 factory.as_ref(),
1222 reconnect,
1223 event_bus.as_ref(),
1224 &mut shutdown,
1225 &control,
1226 &connection_health,
1227 &mut outage_reported,
1228 &mut maintenance_push_rx,
1229 maintenance_enabled,
1230 )
1231 .await
1232 {
1233 return;
1234 }
1235 continue;
1236 }
1237 PendingHandoffOutcome::Ready { sequence } => {
1238 if let Some(state) = &mut maintenance_state {
1239 state.pending = None;
1240 }
1241 connection_health.set(false);
1242 publish_worker_outage(
1243 event_bus.as_ref(),
1244 &mut outage_reported,
1245 ConnectionDisconnectReason::MaintenanceHandoff { sequence },
1246 &control,
1247 );
1248 let ConnSource::Factory { factory, reconnect } = &source else {
1249 fail_queued_requests(&mut rx);
1250 return;
1251 };
1252 let maintenance_enabled = maintenance_state
1253 .as_ref()
1254 .is_some_and(|state| state.enabled);
1255 if !reconnect_worker_connection(
1256 conn.as_mut()
1257 .expect("maintenance listener requires a connection"),
1258 &mut rx,
1259 factory.as_ref(),
1260 reconnect,
1261 event_bus.as_ref(),
1262 &mut shutdown,
1263 &control,
1264 &connection_health,
1265 &mut outage_reported,
1266 &mut maintenance_push_rx,
1267 maintenance_enabled,
1268 )
1269 .await
1270 {
1271 return;
1272 }
1273 continue;
1274 }
1275 }
1276 }
1277
1278 let first = if shutting_down {
1282 rx.try_recv().ok()
1287 } else {
1288 loop {
1289 tokio::select! {
1290 biased;
1291 () = wait_for_shutdown(&mut shutdown) => {
1292 shutting_down = true;
1293 rx.close();
1294 break rx.try_recv().ok();
1295 }
1296 control_message = recv_maintenance_control(&mut maintenance_control_rx) => {
1297 apply_maintenance_control(
1298 control_message,
1299 &mut maintenance_state,
1300 &mut maintenance_control_rx,
1301 &mut maintenance_push_rx,
1302 );
1303 }
1304 push = recv_maintenance_push(&mut maintenance_push_rx) => {
1305 if handle_maintenance_push_result(
1306 push,
1307 &mut maintenance_state,
1308 &mut maintenance_push_rx,
1309 event_bus.as_ref(),
1310 ).is_err() {
1311 connection_health.set(false);
1312 let failure = PipelineFailure::Connection(
1313 maintenance_feed_error(),
1314 );
1315 publish_worker_outage(
1316 event_bus.as_ref(),
1317 &mut outage_reported,
1318 failure.event_reason(),
1319 &control,
1320 );
1321 let ConnSource::Factory { factory, reconnect } = &source else {
1322 fail_queued_requests(&mut rx);
1323 return;
1324 };
1325 let maintenance_enabled = maintenance_state
1326 .as_ref()
1327 .is_some_and(|state| state.enabled);
1328 if !reconnect_worker_connection(
1329 conn.as_mut().expect("maintenance feed requires a connection"),
1330 &mut rx,
1331 factory.as_ref(),
1332 reconnect,
1333 event_bus.as_ref(),
1334 &mut shutdown,
1335 &control,
1336 &connection_health,
1337 &mut outage_reported,
1338 &mut maintenance_push_rx,
1339 maintenance_enabled,
1340 ).await {
1341 return;
1342 }
1343 }
1344 if maintenance_state
1345 .as_ref()
1346 .and_then(|state| state.pending.as_ref())
1347 .is_some()
1348 {
1349 continue 'worker;
1350 }
1351 }
1352 request = rx.recv() => break request,
1353 idle_result = read_idle_push_if_connected(&mut conn) => match idle_result {
1354 Ok(()) => {
1355 }
1359 Err(error) => {
1360 clear_pending_handoff(&mut maintenance_state);
1361 connection_health.set(false);
1362 let failure = PipelineFailure::Connection(error);
1363 publish_worker_outage(
1364 event_bus.as_ref(),
1365 &mut outage_reported,
1366 failure.event_reason(),
1367 &control,
1368 );
1369 match &source {
1370 ConnSource::Fixed => {
1371 rx.close();
1376 while let Ok(request) = rx.try_recv() {
1377 request.fail(RedisError::ConnectionClosed);
1378 }
1379 return;
1380 }
1381 ConnSource::Factory { factory, reconnect } => {
1382 let maintenance_enabled = maintenance_state
1383 .as_ref()
1384 .is_some_and(|state| state.enabled);
1385 if !reconnect_worker_connection(
1386 conn.as_mut().expect("idle failure requires a connection"),
1387 &mut rx,
1388 factory.as_ref(),
1389 reconnect,
1390 event_bus.as_ref(),
1391 &mut shutdown,
1392 &control,
1393 &connection_health,
1394 &mut outage_reported,
1395 &mut maintenance_push_rx,
1396 maintenance_enabled,
1397 )
1398 .await
1399 {
1400 return;
1401 }
1402 }
1403 }
1404 }
1405 },
1406 }
1407 }
1408 };
1409 let first = match first {
1410 Some(req) => req,
1411 None => break, };
1413
1414 let mut frame_count = first.frame_count();
1415 let mut batch: Vec<WorkerRequest> = vec![first];
1416
1417 while frame_count < config.max_batch_size {
1421 match rx.try_recv() {
1422 Ok(req) => {
1423 frame_count += req.frame_count();
1424 batch.push(req);
1425 }
1426 Err(_) => break,
1427 }
1428 }
1429
1430 let mut channel_closed = false;
1433 if frame_count < config.max_batch_size && !config.batch_window.is_zero() {
1434 let deadline = tokio::time::Instant::now() + config.batch_window;
1435 loop {
1436 if frame_count >= config.max_batch_size {
1437 break;
1438 }
1439
1440 if shutting_down {
1441 match rx.try_recv() {
1442 Ok(req) => {
1443 frame_count += req.frame_count();
1444 batch.push(req);
1445 continue;
1446 }
1447 Err(_) => break,
1448 }
1449 }
1450
1451 tokio::select! {
1452 biased;
1453 () = wait_for_shutdown(&mut shutdown) => {
1454 shutting_down = true;
1455 rx.close();
1456 }
1457 request = rx.recv() => match request {
1458 Some(req) => {
1459 frame_count += req.frame_count();
1460 batch.push(req);
1461 }
1462 None => {
1463 channel_closed = true;
1464 break;
1465 }
1466 },
1467 () = tokio::time::sleep_until(deadline) => break,
1468 }
1469 }
1470 }
1471
1472 if conn.is_none() {
1473 let ConnSource::Factory { factory, reconnect } = &source else {
1474 unreachable!("a fixed worker always starts with a connection");
1475 };
1476 let connect = connect_with_timeout(factory.as_ref(), reconnect.connect_timeout);
1477 tokio::pin!(connect);
1478 let connect_result = tokio::select! {
1479 biased;
1480 () = wait_for_shutdown(&mut shutdown) => {
1481 shutting_down = true;
1482 rx.close();
1483 fail_requests(batch);
1484 continue;
1485 }
1486 result = &mut connect => result,
1487 };
1488 match connect_result {
1489 Ok(new_conn) => {
1490 let connected = control.with_active_handle(|| {
1491 connection_health.set(true);
1492 if let Some(events) = &event_bus {
1493 events.publish(ConnectionEvent::Connected);
1494 }
1495 });
1496 if connected.is_none() {
1497 fail_requests(batch);
1498 return;
1499 }
1500 conn = Some(new_conn);
1501 }
1502 Err(error) => {
1503 control.with_active_handle(|| {
1504 if let Some(events) = &event_bus {
1505 events.publish_with(|| ConnectionEvent::ConnectFailed {
1506 error: Arc::from(error.to_string()),
1507 });
1508 }
1509 });
1510 fail_requests(batch);
1511 continue;
1512 }
1513 }
1514 }
1515
1516 let flush_result = flush_batch(
1517 conn.as_mut().expect("connection established before flush"),
1518 batch,
1519 config.response_timeout,
1520 config.metrics_recorder.as_ref(),
1521 config.reconnect_on_readonly,
1522 &connection_health,
1523 )
1524 .await;
1525
1526 if channel_closed {
1527 if let Err(failure) = flush_result {
1528 publish_worker_outage(
1529 event_bus.as_ref(),
1530 &mut outage_reported,
1531 failure.event_reason(),
1532 &control,
1533 );
1534 }
1535 return;
1536 }
1537
1538 if let Err(failure) = flush_result {
1539 drain_maintenance_control(
1544 &mut maintenance_state,
1545 &mut maintenance_control_rx,
1546 &mut maintenance_push_rx,
1547 );
1548 drain_available_maintenance_pushes(
1549 &mut maintenance_state,
1550 &mut maintenance_push_rx,
1551 event_bus.as_ref(),
1552 );
1553 clear_pending_handoff(&mut maintenance_state);
1554 publish_worker_outage(
1555 event_bus.as_ref(),
1556 &mut outage_reported,
1557 failure.event_reason(),
1558 &control,
1559 );
1560 match &source {
1563 ConnSource::Fixed => {
1564 }
1568 ConnSource::Factory { factory, reconnect } => {
1569 let maintenance_enabled = maintenance_state
1570 .as_ref()
1571 .is_some_and(|state| state.enabled);
1572 if !reconnect_worker_connection(
1573 conn.as_mut().expect("failed flush requires a connection"),
1574 &mut rx,
1575 factory.as_ref(),
1576 reconnect,
1577 event_bus.as_ref(),
1578 &mut shutdown,
1579 &control,
1580 &connection_health,
1581 &mut outage_reported,
1582 &mut maintenance_push_rx,
1583 maintenance_enabled,
1584 )
1585 .await
1586 {
1587 return;
1588 }
1589 }
1590 }
1591 }
1592 }
1593}
1594
1595async fn read_idle_push_if_connected(conn: &mut Option<RedisConnection>) -> Result<(), RedisError> {
1601 match conn {
1602 Some(conn) => conn.read_idle_push().await,
1603 None => futures::future::pending().await,
1604 }
1605}
1606
1607enum PendingHandoffOutcome {
1608 Cancelled,
1609 Ready { sequence: u64 },
1610 ConnectionFailed(RedisError),
1611 Shutdown,
1612}
1613
1614async fn recv_maintenance_control(
1615 receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1616) -> Option<MaintenanceControl> {
1617 match receiver {
1618 Some(receiver) => receiver.recv().await,
1619 None => futures::future::pending().await,
1620 }
1621}
1622
1623async fn recv_maintenance_push(
1624 receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1625) -> Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError> {
1626 match receiver {
1627 Some(receiver) => receiver.recv().await,
1628 None => futures::future::pending().await,
1629 }
1630}
1631
1632fn drain_maintenance_control(
1633 state: &mut Option<MaintenanceState>,
1634 receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1635 pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1636) {
1637 loop {
1638 let Some(control) = receiver.as_mut() else {
1639 return;
1640 };
1641 match control.try_recv() {
1642 Ok(message) => apply_maintenance_control(Some(message), state, receiver, pushes),
1643 Err(mpsc::error::TryRecvError::Empty) => return,
1644 Err(mpsc::error::TryRecvError::Disconnected) => {
1645 *receiver = None;
1646 if let Some(state) = state {
1647 state.enabled = false;
1648 state.pending = None;
1649 }
1650 *pushes = None;
1651 return;
1652 }
1653 }
1654 }
1655}
1656
1657fn clear_pending_handoff(state: &mut Option<MaintenanceState>) {
1658 if let Some(state) = state {
1659 state.pending = None;
1660 }
1661}
1662
1663fn disable_maintenance(
1664 state: &mut Option<MaintenanceState>,
1665 receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1666 pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1667) {
1668 if let Some(state) = state {
1669 state.enabled = false;
1670 state.pending = None;
1671 }
1672 *receiver = None;
1673 *pushes = None;
1674}
1675
1676fn apply_maintenance_control(
1677 message: Option<MaintenanceControl>,
1678 state: &mut Option<MaintenanceState>,
1679 receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1680 pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1681) {
1682 let Some(message) = message else {
1683 *receiver = None;
1684 return;
1685 };
1686 match message {
1687 MaintenanceControl::Disable { listener_id, ack } => {
1688 if state
1689 .as_ref()
1690 .is_some_and(|state| state.listener_id == listener_id)
1691 {
1692 disable_maintenance(state, receiver, pushes);
1693 }
1694 if let Some(ack) = ack {
1695 let _ = ack.send(());
1696 }
1697 }
1698 }
1699}
1700
1701fn publish_maintenance_notification(
1702 event_bus: Option<&ConnectionEventBus>,
1703 notification: ParsedMaintenanceNotification,
1704) {
1705 if let Some(events) = event_bus {
1706 events.publish(ConnectionEvent::MaintenanceNotification {
1707 kind: notification.kind(),
1708 sequence: notification.sequence(),
1709 ttl: notification.ttl(),
1710 });
1711 }
1712}
1713
1714fn handle_maintenance_push_result(
1715 push: Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError>,
1716 state: &mut Option<MaintenanceState>,
1717 receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1718 event_bus: Option<&ConnectionEventBus>,
1719) -> Result<(), ()> {
1720 match push {
1721 Ok(push) => {
1722 if let Some(state) = state
1723 && let Some(received) = parse_maintenance_push(push)
1724 && let Some(notification) = state.accept(received)
1725 {
1726 publish_maintenance_notification(event_bus, notification);
1727 }
1728 Ok(())
1729 }
1730 Err(_) => {
1731 *receiver = None;
1732 Err(())
1733 }
1734 }
1735}
1736
1737fn drain_available_maintenance_pushes(
1738 state: &mut Option<MaintenanceState>,
1739 receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1740 event_bus: Option<&ConnectionEventBus>,
1741) {
1742 loop {
1743 let Some(pushes) = receiver.as_mut() else {
1744 return;
1745 };
1746 match pushes.try_recv() {
1747 Ok(push) => {
1748 let _ = handle_maintenance_push_result(Ok(push), state, receiver, event_bus);
1749 }
1750 Err(tokio::sync::broadcast::error::TryRecvError::Empty) => return,
1751 Err(tokio::sync::broadcast::error::TryRecvError::Lagged(_)) => {
1752 }
1756 Err(tokio::sync::broadcast::error::TryRecvError::Closed) => {
1757 *receiver = None;
1758 return;
1759 }
1760 }
1761 }
1762}
1763
1764fn maintenance_feed_error() -> RedisError {
1765 RedisError::UnexpectedResponse {
1766 expected: "a complete maintenance-notification push feed",
1767 actual: "maintenance push feed closed or overflowed".to_owned(),
1768 }
1769}
1770
1771async fn connect_and_prepare(
1772 factory: &dyn ConnectionFactory,
1773 timeout: Option<Duration>,
1774 maintenance_enabled: bool,
1775) -> Result<
1776 (
1777 RedisConnection,
1778 Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1779 ),
1780 RedisError,
1781> {
1782 let prepare = async {
1783 let mut connection = factory.connect().await?;
1784 let pushes = if maintenance_enabled {
1785 Some(register_connection(&mut connection).await?)
1786 } else {
1787 None
1788 };
1789 Ok((connection, pushes))
1790 };
1791 match timeout {
1792 Some(timeout) => tokio::time::timeout(timeout, prepare)
1793 .await
1794 .map_err(|_| RedisError::ConnectTimeout)?,
1795 None => prepare.await,
1796 }
1797}
1798
1799async fn wait_for_pending_handoff(
1800 conn: &mut Option<RedisConnection>,
1801 state: &mut Option<MaintenanceState>,
1802 control_rx: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1803 push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1804 event_bus: Option<&ConnectionEventBus>,
1805 shutdown: &mut watch::Receiver<bool>,
1806) -> PendingHandoffOutcome {
1807 loop {
1808 let Some(pending) = state.as_ref().and_then(|state| state.pending.as_ref()) else {
1809 return PendingHandoffOutcome::Cancelled;
1810 };
1811 let deadline = pending.deadline;
1812 let sequence = pending.sequence;
1813 if deadline <= tokio::time::Instant::now() {
1814 return PendingHandoffOutcome::Ready { sequence };
1815 }
1816
1817 tokio::select! {
1818 biased;
1819 () = wait_for_shutdown(shutdown) => return PendingHandoffOutcome::Shutdown,
1820 control = recv_maintenance_control(control_rx) => {
1821 apply_maintenance_control(control, state, control_rx, push_rx);
1822 if state.as_ref().and_then(|state| state.pending.as_ref()).is_none() {
1823 return PendingHandoffOutcome::Cancelled;
1824 }
1825 }
1826 push = recv_maintenance_push(push_rx) => {
1827 if handle_maintenance_push_result(push, state, push_rx, event_bus).is_err() {
1828 return PendingHandoffOutcome::ConnectionFailed(maintenance_feed_error());
1829 }
1830 }
1831 result = read_idle_push_if_connected(conn) => {
1832 if let Err(error) = result {
1833 return PendingHandoffOutcome::ConnectionFailed(error);
1834 }
1835 }
1836 () = tokio::time::sleep_until(deadline) => {
1837 }
1840 }
1841 }
1842}
1843
1844fn fail_requests(requests: Vec<WorkerRequest>) {
1845 for request in requests {
1846 request.fail(RedisError::ConnectionClosed);
1847 }
1848}
1849
1850#[allow(clippy::too_many_arguments)]
1856async fn reconnect_worker_connection(
1857 conn: &mut RedisConnection,
1858 rx: &mut mpsc::Receiver<WorkerRequest>,
1859 factory: &dyn ConnectionFactory,
1860 reconnect: &ReconnectConfig,
1861 event_bus: Option<&ConnectionEventBus>,
1862 shutdown: &mut watch::Receiver<bool>,
1863 control: &WorkerControl,
1864 connection_health: &ConnectionHealthPublisher,
1865 outage_reported: &mut bool,
1866 maintenance_push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1867 maintenance_enabled: bool,
1868) -> bool {
1869 let started = Instant::now();
1870 match reconnect_with_backoff(
1871 factory,
1872 reconnect,
1873 event_bus,
1874 shutdown,
1875 control,
1876 maintenance_enabled,
1877 )
1878 .await
1879 {
1880 ReconnectOutcome::Connected(new_conn, new_push_rx, attempts) => {
1881 let reconnected = control.with_active_handle(|| {
1882 *conn = *new_conn;
1883 *maintenance_push_rx = new_push_rx;
1884 connection_health.set(true);
1887 *outage_reported = false;
1888 if let Some(events) = event_bus {
1889 events.publish(ConnectionEvent::Reconnected {
1890 attempts,
1891 elapsed: started.elapsed(),
1892 });
1893 }
1894 });
1895 if reconnected.is_some() {
1896 true
1897 } else {
1898 fail_queued_requests(rx);
1899 false
1900 }
1901 }
1902 ReconnectOutcome::Exhausted | ReconnectOutcome::Shutdown => {
1903 fail_queued_requests(rx);
1904 false
1905 }
1906 }
1907}
1908
1909fn fail_queued_requests(rx: &mut mpsc::Receiver<WorkerRequest>) {
1910 rx.close();
1911 while let Ok(request) = rx.try_recv() {
1912 request.fail(RedisError::ConnectionClosed);
1913 }
1914}
1915
1916fn publish_worker_outage(
1917 event_bus: Option<&ConnectionEventBus>,
1918 outage_reported: &mut bool,
1919 reason: ConnectionDisconnectReason,
1920 control: &WorkerControl,
1921) {
1922 if *outage_reported {
1923 return;
1924 }
1925 let active = control.with_active_handle(|| {
1926 if let Some(events) = event_bus {
1927 events.publish(ConnectionEvent::Disconnected { reason });
1928 }
1929 });
1930 if active.is_some() {
1931 *outage_reported = true;
1932 }
1933}
1934
1935fn is_readonly_frame(frame: &Frame) -> bool {
1938 matches!(frame, Frame::Error(e) if e.len() >= 8 && e[..8].eq_ignore_ascii_case(b"READONLY"))
1939}
1940
1941async fn flush_batch(
1959 conn: &mut RedisConnection,
1960 batch: Vec<WorkerRequest>,
1961 response_timeout: Option<Duration>,
1962 recorder: Option<&Arc<dyn MetricsRecorder>>,
1963 reconnect_on_readonly: bool,
1964 connection_health: &ConnectionHealthPublisher,
1965) -> Result<(), PipelineFailure> {
1966 let mut batch = batch;
1971 batch.retain(|request| !request.response_is_closed());
1972 if batch.is_empty() {
1973 return Ok(());
1974 }
1975
1976 let total_frames: usize = batch.iter().map(|r| r.frame_count()).sum();
1982
1983 if let Some(recorder) = recorder {
1986 recorder.pipeline_flushed(total_frames);
1987 }
1988
1989 let mut frames: Vec<Frame> = Vec::with_capacity(total_frames);
1990
1991 enum Responder {
1992 Single(oneshot::Sender<Result<Frame, RedisError>>),
1993 Multi(usize, oneshot::Sender<Result<Vec<Frame>, RedisError>>),
1994 }
1995 let mut responders: Vec<Responder> = Vec::with_capacity(batch.len());
1996
1997 for req in batch {
1998 match req {
1999 WorkerRequest::Single { frame, response_tx } => {
2000 frames.push(frame);
2001 responders.push(Responder::Single(response_tx));
2002 }
2003 WorkerRequest::Multi {
2004 frames: fs,
2005 response_tx,
2006 } => {
2007 let count = fs.len();
2008 frames.extend(fs);
2009 responders.push(Responder::Multi(count, response_tx));
2010 }
2011 }
2012 }
2013
2014 let exec_result = match response_timeout {
2015 Some(timeout) => match tokio::time::timeout(timeout, conn.execute_pipeline(frames)).await {
2016 Ok(result) => result,
2017 Err(_elapsed) => {
2018 connection_health.set(false);
2023 for responder in responders {
2024 match responder {
2025 Responder::Single(tx) => {
2026 let _ = tx.send(Err(RedisError::CommandTimeout));
2027 }
2028 Responder::Multi(_, tx) => {
2029 let _ = tx.send(Err(RedisError::CommandTimeout));
2030 }
2031 }
2032 }
2033 return Err(PipelineFailure::CommandTimeout);
2034 }
2035 },
2036 None => conn.execute_pipeline(frames).await,
2037 };
2038 match exec_result {
2039 Ok(responses) => {
2040 let saw_readonly = reconnect_on_readonly && responses.iter().any(is_readonly_frame);
2045
2046 if saw_readonly {
2047 connection_health.set(false);
2048 }
2049
2050 let mut iter = responses.into_iter();
2051 for responder in responders {
2052 match responder {
2053 Responder::Single(tx) => {
2054 let _ = tx.send(iter.next().ok_or(RedisError::ConnectionClosed));
2055 }
2056 Responder::Multi(count, tx) => {
2057 let collected: Vec<Frame> = iter.by_ref().take(count).collect();
2058 if collected.len() == count {
2059 let _ = tx.send(Ok(collected));
2060 } else {
2061 let _ = tx.send(Err(RedisError::ConnectionClosed));
2062 }
2063 }
2064 }
2065 }
2066 if saw_readonly {
2067 Err(PipelineFailure::ReadOnly)
2068 } else {
2069 Ok(())
2070 }
2071 }
2072 Err(error) => {
2073 connection_health.set(false);
2074 for responder in responders {
2075 match responder {
2076 Responder::Single(tx) => {
2077 let _ = tx.send(Err(RedisError::ConnectionClosed));
2078 }
2079 Responder::Multi(_, tx) => {
2080 let _ = tx.send(Err(RedisError::ConnectionClosed));
2081 }
2082 }
2083 }
2084 Err(PipelineFailure::Connection(error))
2085 }
2086 }
2087}
2088
2089enum ReconnectOutcome {
2090 Connected(
2091 Box<RedisConnection>,
2092 Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
2093 usize,
2094 ),
2095 Exhausted,
2096 Shutdown,
2097}
2098
2099async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
2100 if *shutdown.borrow() {
2101 return;
2102 }
2103 let _ = shutdown.changed().await;
2104}
2105
2106async fn reconnect_with_backoff(
2109 factory: &dyn ConnectionFactory,
2110 config: &ReconnectConfig,
2111 event_bus: Option<&ConnectionEventBus>,
2112 shutdown: &mut watch::Receiver<bool>,
2113 control: &WorkerControl,
2114 maintenance_enabled: bool,
2115) -> ReconnectOutcome {
2116 let mut attempt: usize = 0;
2117 loop {
2118 if *shutdown.borrow() {
2119 return ReconnectOutcome::Shutdown;
2120 }
2121 if config.attempt_exhausted(attempt) {
2122 warn!(
2123 attempts = attempt,
2124 "auto_pipeline: reconnect attempts exhausted"
2125 );
2126 let active = control.with_active_handle(|| {
2127 if let Some(events) = event_bus {
2128 events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
2129 }
2130 });
2131 if active.is_none() {
2132 return ReconnectOutcome::Shutdown;
2133 }
2134 return ReconnectOutcome::Exhausted;
2135 }
2136 let delay = config.delay_for_attempt(attempt);
2137 let active = control.with_active_handle(|| {
2138 if let Some(events) = event_bus {
2139 events.publish(ConnectionEvent::ReconnectAttempt {
2140 attempt: attempt + 1,
2141 delay,
2142 });
2143 }
2144 });
2145 if active.is_none() {
2146 return ReconnectOutcome::Shutdown;
2147 }
2148 if !delay.is_zero() {
2149 tokio::select! {
2150 biased;
2151 () = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
2152 () = tokio::time::sleep(delay) => {}
2153 }
2154 }
2155 attempt += 1;
2156 let connect = connect_and_prepare(factory, config.connect_timeout, maintenance_enabled);
2157 tokio::pin!(connect);
2158 let result = tokio::select! {
2159 biased;
2160 () = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
2161 result = &mut connect => result,
2162 };
2163 match result {
2164 Ok((conn, push_rx)) => {
2165 return ReconnectOutcome::Connected(Box::new(conn), push_rx, attempt);
2166 }
2167 Err(e) => {
2168 warn!(attempt, error = %e, "auto_pipeline: reconnect attempt failed");
2169 let active = control.with_active_handle(|| {
2170 if let Some(events) = event_bus {
2171 events.publish_with(|| ConnectionEvent::ReconnectFailed {
2172 attempt,
2173 error: Arc::from(e.to_string()),
2174 });
2175 }
2176 });
2177 if active.is_none() {
2178 return ReconnectOutcome::Shutdown;
2179 }
2180 continue;
2181 }
2182 }
2183 }
2184}
2185
2186impl Service<Frame> for AutoPipelineService {
2187 type Response = Frame;
2188 type Error = RedisError;
2189 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
2190
2191 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
2192 if self.lease.control.is_shutdown_requested() {
2193 let _ = self.poll_tx.abort_send();
2194 return Poll::Ready(Err(RedisError::ConnectionClosed));
2195 }
2196 let shutdown_poll = self.shutdown_notified.as_mut().poll(cx);
2200 if shutdown_poll.is_ready() || self.lease.control.is_shutdown_requested() {
2201 let _ = self.poll_tx.abort_send();
2202 return Poll::Ready(Err(RedisError::ConnectionClosed));
2203 }
2204 if self.shed_load {
2205 return if self.tx.is_closed() {
2208 Poll::Ready(Err(RedisError::ConnectionClosed))
2209 } else {
2210 Poll::Ready(Ok(()))
2211 };
2212 }
2213 self.poll_tx
2215 .poll_reserve(cx)
2216 .map_err(|_| RedisError::ConnectionClosed)
2217 }
2218
2219 fn call(&mut self, frame: Frame) -> Self::Future {
2220 let (resp_tx, resp_rx) = oneshot::channel();
2221 let request = WorkerRequest::Single {
2222 frame,
2223 response_tx: resp_tx,
2224 };
2225
2226 if self.shed_load {
2227 let send_result = self
2232 .lease
2233 .control
2234 .with_admission(|| {
2235 self.tx.try_send(request).map_err(|e| match e {
2236 tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
2237 tokio::sync::mpsc::error::TrySendError::Closed(_) => {
2238 RedisError::ConnectionClosed
2239 }
2240 })
2241 })
2242 .unwrap_or(Err(RedisError::ConnectionClosed));
2243 return Box::pin(async move {
2244 send_result?;
2245 resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
2246 });
2247 }
2248
2249 let send_result = self
2252 .lease
2253 .control
2254 .with_admission(|| {
2255 self.poll_tx
2256 .send_item(request)
2257 .map_err(|_| RedisError::ConnectionClosed)
2258 })
2259 .unwrap_or_else(|| {
2260 let _ = self.poll_tx.abort_send();
2261 Err(RedisError::ConnectionClosed)
2262 });
2263 Box::pin(async move {
2264 send_result?;
2265 resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
2266 })
2267 }
2268}
2269
2270impl Clone for AutoPipelineService {
2271 fn clone(&self) -> Self {
2272 Self {
2274 tx: self.tx.clone(),
2275 poll_tx: PollSender::new(self.tx.clone()),
2276 shed_load: self.shed_load,
2277 lease: self.lease.clone(),
2278 shutdown_notified: Box::pin(self.lease.control.shutdown_notified()),
2279 connection_health: self.connection_health.clone(),
2280 worker: Arc::clone(&self.worker),
2281 }
2282 }
2283}
2284
2285impl AutoPipelineService {
2286 pub fn queue_depth(&self) -> usize {
2291 self.tx.max_capacity() - self.tx.capacity()
2292 }
2293}
2294
2295#[cfg(test)]
2296mod tests {
2297 use super::*;
2298
2299 #[test]
2300 fn lazy_factory_requires_an_entered_tokio_runtime() {
2301 let result = std::panic::catch_unwind(|| {
2302 AutoPipelineService::with_lazy_factory(
2303 || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) },
2304 AutoPipelineConfig::default(),
2305 AutoPipelineReconnectConfig::default(),
2306 )
2307 });
2308 assert!(result.is_err());
2309 }
2310
2311 #[tokio::test]
2312 async fn lazy_factory_defers_connection_and_retries_after_initial_failure() {
2313 use futures::{SinkExt, StreamExt};
2314 use std::sync::atomic::{AtomicUsize, Ordering};
2315 use tokio_util::codec::Framed;
2316
2317 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2318 let addr = listener.local_addr().unwrap();
2319 let server = tokio::spawn(async move {
2320 let (stream, _) = listener.accept().await.unwrap();
2321 let mut framed = Framed::new(
2322 redis_tower_core::RedisStream::Tcp(stream),
2323 redis_tower_core::RespCodec::new(),
2324 );
2325 let request = framed
2326 .next()
2327 .await
2328 .expect("client closed before request")
2329 .expect("client sent an invalid request");
2330 assert_eq!(
2331 request,
2332 Frame::SimpleString(bytes::Bytes::from_static(b"PING"))
2333 );
2334 framed
2335 .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
2336 .await
2337 .unwrap();
2338 let _ = framed.next().await;
2339 });
2340
2341 let calls = Arc::new(AtomicUsize::new(0));
2342 let factory_calls = Arc::clone(&calls);
2343 let factory = move || {
2344 let factory_calls = Arc::clone(&factory_calls);
2345 async move {
2346 if factory_calls.fetch_add(1, Ordering::AcqRel) == 0 {
2347 return Err(RedisError::ConnectionClosed);
2348 }
2349 let stream = tokio::net::TcpStream::connect(addr)
2350 .await
2351 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2352 Ok(RedisConnection::from_stream(
2353 redis_tower_core::RedisStream::Tcp(stream),
2354 ))
2355 }
2356 };
2357
2358 let events = ConnectionEventBus::new(8);
2359 let mut event_stream = events.subscribe();
2360 let mut service = AutoPipelineService::with_lazy_factory_and_events(
2361 factory,
2362 AutoPipelineConfig::default(),
2363 AutoPipelineReconnectConfig::default(),
2364 events,
2365 );
2366
2367 assert_eq!(calls.load(Ordering::Acquire), 0);
2368 assert!(!service.is_connection_healthy());
2369 assert!(
2370 tokio::time::timeout(Duration::from_millis(10), event_stream.recv())
2371 .await
2372 .is_err(),
2373 "lazy construction must not publish a connection event"
2374 );
2375
2376 futures::future::poll_fn(|cx| service.poll_ready(cx))
2377 .await
2378 .unwrap();
2379 assert!(matches!(
2380 service
2381 .call(Frame::SimpleString(bytes::Bytes::from_static(b"FIRST")))
2382 .await,
2383 Err(RedisError::ConnectionClosed)
2384 ));
2385 assert!(!service.is_connection_healthy());
2386 assert!(matches!(
2387 event_stream.recv().await.unwrap(),
2388 ConnectionEvent::ConnectFailed { .. }
2389 ));
2390
2391 futures::future::poll_fn(|cx| service.poll_ready(cx))
2392 .await
2393 .unwrap();
2394 assert_eq!(
2395 service
2396 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
2397 .await
2398 .unwrap(),
2399 Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
2400 );
2401 assert!(service.is_connection_healthy());
2402 assert_eq!(
2403 event_stream.recv().await.unwrap(),
2404 ConnectionEvent::Connected
2405 );
2406 assert_eq!(calls.load(Ordering::Acquire), 2);
2407
2408 service.shutdown().await;
2409 server.await.unwrap();
2410 }
2411
2412 #[tokio::test]
2413 async fn lazy_factory_shutdown_cancels_first_connection_attempt() {
2414 use tokio::sync::Notify;
2415
2416 let entered = Arc::new(Notify::new());
2417 let factory_entered = Arc::clone(&entered);
2418 let factory = move || {
2419 let factory_entered = Arc::clone(&factory_entered);
2420 async move {
2421 factory_entered.notify_one();
2422 futures::future::pending::<Result<RedisConnection, RedisError>>().await
2423 }
2424 };
2425 let mut service = AutoPipelineService::with_lazy_factory(
2426 factory,
2427 AutoPipelineConfig {
2428 shed_load_on_full: true,
2429 ..AutoPipelineConfig::default()
2430 },
2431 AutoPipelineReconnectConfig::default(),
2432 );
2433
2434 let response = service.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")));
2435 entered.notified().await;
2436 tokio::time::timeout(Duration::from_secs(1), service.shutdown())
2437 .await
2438 .expect("shutdown must cancel a deferred connection attempt");
2439 assert!(matches!(response.await, Err(RedisError::ConnectionClosed)));
2440 }
2441
2442 #[tokio::test]
2443 async fn initial_factory_timeout_publishes_connect_failed() {
2444 let factory =
2445 || async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
2446 let events = ConnectionEventBus::new(4);
2447 let mut stream = events.subscribe();
2448 let reconnect = AutoPipelineReconnectConfig::new(
2449 ReconnectConfig::default().connect_timeout(Duration::from_millis(10)),
2450 );
2451
2452 let result = AutoPipelineService::with_factory_and_events(
2453 factory,
2454 AutoPipelineConfig::default(),
2455 reconnect,
2456 events,
2457 )
2458 .await;
2459 assert!(matches!(result, Err(RedisError::ConnectTimeout)));
2460 assert_eq!(
2461 stream.recv().await.unwrap(),
2462 ConnectionEvent::ConnectFailed {
2463 error: Arc::from(RedisError::ConnectTimeout.to_string()),
2464 }
2465 );
2466 }
2467
2468 #[tokio::test]
2469 async fn reconnect_factory_timeout_is_counted_and_exhausted() {
2470 let factory =
2471 || async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
2472 let config = ReconnectConfig {
2473 max_retries: Some(0),
2474 base_delay: Duration::ZERO,
2475 max_delay: Duration::ZERO,
2476 jitter: false,
2477 connect_timeout: Some(Duration::from_millis(10)),
2478 };
2479 let events = ConnectionEventBus::new(4);
2480 let mut stream = events.subscribe();
2481 let (shutdown_tx, mut shutdown) = watch::channel(false);
2482 let control = WorkerControl::new(shutdown_tx, None);
2483
2484 assert!(matches!(
2485 reconnect_with_backoff(
2486 &factory,
2487 &config,
2488 Some(&events),
2489 &mut shutdown,
2490 &control,
2491 false,
2492 )
2493 .await,
2494 ReconnectOutcome::Exhausted
2495 ));
2496 assert_eq!(
2497 stream.recv().await.unwrap(),
2498 ConnectionEvent::ReconnectAttempt {
2499 attempt: 1,
2500 delay: Duration::ZERO,
2501 }
2502 );
2503 assert_eq!(
2504 stream.recv().await.unwrap(),
2505 ConnectionEvent::ReconnectFailed {
2506 attempt: 1,
2507 error: Arc::from(RedisError::ConnectTimeout.to_string()),
2508 }
2509 );
2510 assert_eq!(
2511 stream.recv().await.unwrap(),
2512 ConnectionEvent::ReconnectExhausted { attempts: 1 }
2513 );
2514 }
2515
2516 #[tokio::test]
2517 async fn reconnect_exhaustion_is_published_after_final_failure() {
2518 let factory = || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) };
2519 let config = ReconnectConfig {
2520 max_retries: Some(2),
2521 base_delay: Duration::ZERO,
2522 max_delay: Duration::ZERO,
2523 jitter: false,
2524 connect_timeout: None,
2525 };
2526 let events = ConnectionEventBus::new(8);
2527 let mut stream = events.subscribe();
2528 let (shutdown_tx, mut shutdown) = watch::channel(false);
2529 let control = WorkerControl::new(shutdown_tx, None);
2530
2531 assert!(matches!(
2532 reconnect_with_backoff(
2533 &factory,
2534 &config,
2535 Some(&events),
2536 &mut shutdown,
2537 &control,
2538 false,
2539 )
2540 .await,
2541 ReconnectOutcome::Exhausted
2542 ));
2543
2544 assert_eq!(
2545 stream.recv().await.unwrap(),
2546 ConnectionEvent::ReconnectAttempt {
2547 attempt: 1,
2548 delay: Duration::ZERO,
2549 }
2550 );
2551 assert!(matches!(
2552 stream.recv().await.unwrap(),
2553 ConnectionEvent::ReconnectFailed { attempt: 1, .. }
2554 ));
2555 assert_eq!(
2556 stream.recv().await.unwrap(),
2557 ConnectionEvent::ReconnectAttempt {
2558 attempt: 2,
2559 delay: Duration::ZERO,
2560 }
2561 );
2562 assert!(matches!(
2563 stream.recv().await.unwrap(),
2564 ConnectionEvent::ReconnectFailed { attempt: 2, .. }
2565 ));
2566 assert_eq!(
2567 stream.recv().await.unwrap(),
2568 ConnectionEvent::ReconnectAttempt {
2569 attempt: 3,
2570 delay: Duration::ZERO,
2571 }
2572 );
2573 assert!(matches!(
2574 stream.recv().await.unwrap(),
2575 ConnectionEvent::ReconnectFailed { attempt: 3, .. }
2576 ));
2577 assert_eq!(
2578 stream.recv().await.unwrap(),
2579 ConnectionEvent::ReconnectExhausted { attempts: 3 }
2580 );
2581 }
2582
2583 #[tokio::test]
2584 async fn factory_worker_detects_idle_disconnect_and_reconnects_in_order() {
2585 use std::sync::atomic::{AtomicUsize, Ordering};
2586 use tokio::sync::{Notify, oneshot};
2587
2588 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2589 let addr = listener.local_addr().unwrap();
2590 let (first_closed_tx, first_closed_rx) = oneshot::channel();
2591 let server = tokio::spawn(async move {
2592 let (first, _) = listener.accept().await.unwrap();
2593 drop(first);
2594 let _ = first_closed_tx.send(());
2595
2596 let (second, _) = listener.accept().await.unwrap();
2597 let _second = second;
2598 futures::future::pending::<()>().await;
2599 });
2600
2601 let calls = Arc::new(AtomicUsize::new(0));
2602 let allow_reconnect = Arc::new(Notify::new());
2603 let factory_calls = Arc::clone(&calls);
2604 let factory_allow_reconnect = Arc::clone(&allow_reconnect);
2605 let factory = move || {
2606 let factory_calls = Arc::clone(&factory_calls);
2607 let factory_allow_reconnect = Arc::clone(&factory_allow_reconnect);
2608 async move {
2609 let call = factory_calls.fetch_add(1, Ordering::AcqRel);
2610 if call == 1 {
2611 return Err(RedisError::ConnectionClosed);
2612 }
2613 if call == 2 {
2614 factory_allow_reconnect.notified().await;
2615 }
2616 let stream = tokio::net::TcpStream::connect(addr)
2617 .await
2618 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2619 Ok(RedisConnection::from_stream(
2620 redis_tower_core::RedisStream::Tcp(stream),
2621 ))
2622 }
2623 };
2624
2625 let events = ConnectionEventBus::new(16);
2626 let mut event_stream = events.subscribe();
2627 let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2628 max_retries: Some(3),
2629 base_delay: Duration::ZERO,
2630 max_delay: Duration::ZERO,
2631 jitter: false,
2632 connect_timeout: None,
2633 });
2634 let service = AutoPipelineService::with_factory_and_events(
2635 factory,
2636 AutoPipelineConfig::default(),
2637 reconnect,
2638 events,
2639 )
2640 .await
2641 .unwrap();
2642 let mut health = service.subscribe_connection_health();
2643 assert_eq!(
2644 event_stream.recv().await.unwrap(),
2645 ConnectionEvent::Connected
2646 );
2647
2648 first_closed_rx.await.unwrap();
2649 assert!(matches!(
2650 event_stream.recv().await.unwrap(),
2651 ConnectionEvent::Disconnected {
2652 reason: ConnectionDisconnectReason::ConnectionError { .. }
2653 }
2654 ));
2655 assert_eq!(
2656 event_stream.recv().await.unwrap(),
2657 ConnectionEvent::ReconnectAttempt {
2658 attempt: 1,
2659 delay: Duration::ZERO,
2660 }
2661 );
2662 assert_eq!(
2663 event_stream.recv().await.unwrap(),
2664 ConnectionEvent::ReconnectFailed {
2665 attempt: 1,
2666 error: Arc::from("connection closed"),
2667 }
2668 );
2669 assert_eq!(
2670 event_stream.recv().await.unwrap(),
2671 ConnectionEvent::ReconnectAttempt {
2672 attempt: 2,
2673 delay: Duration::ZERO,
2674 }
2675 );
2676 health.changed().await.unwrap();
2677 assert!(!*health.borrow_and_update());
2678 assert!(!service.is_connection_healthy());
2679
2680 allow_reconnect.notify_one();
2681 assert!(matches!(
2682 event_stream.recv().await.unwrap(),
2683 ConnectionEvent::Reconnected { attempts: 2, .. }
2684 ));
2685 health.changed().await.unwrap();
2686 assert!(*health.borrow_and_update());
2687 assert!(service.is_connection_healthy());
2688 assert_eq!(calls.load(Ordering::Acquire), 3);
2689
2690 service.shutdown().await;
2691 server.abort();
2692 }
2693
2694 #[tokio::test]
2695 async fn shutdown_handle_cancels_hanging_unlimited_reconnect_with_clones() {
2696 use std::sync::atomic::{AtomicUsize, Ordering};
2697 use tokio::sync::{Notify, oneshot};
2698
2699 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2700 let addr = listener.local_addr().unwrap();
2701 let (first_closed_tx, first_closed_rx) = oneshot::channel();
2702 let server = tokio::spawn(async move {
2703 let (first, _) = listener.accept().await.unwrap();
2704 drop(first);
2705 let _ = first_closed_tx.send(());
2706 });
2707
2708 let calls = Arc::new(AtomicUsize::new(0));
2709 let reconnect_entered = Arc::new(Notify::new());
2710 let factory_calls = Arc::clone(&calls);
2711 let factory_entered = Arc::clone(&reconnect_entered);
2712 let factory = move || {
2713 let factory_calls = Arc::clone(&factory_calls);
2714 let factory_entered = Arc::clone(&factory_entered);
2715 async move {
2716 if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
2717 factory_entered.notify_one();
2718 return futures::future::pending::<Result<RedisConnection, RedisError>>().await;
2719 }
2720 let stream = tokio::net::TcpStream::connect(addr)
2721 .await
2722 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2723 Ok(RedisConnection::from_stream(
2724 redis_tower_core::RedisStream::Tcp(stream),
2725 ))
2726 }
2727 };
2728
2729 let events = ConnectionEventBus::new(8);
2730 let mut event_stream = events.subscribe();
2731 let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2732 max_retries: None,
2733 base_delay: Duration::ZERO,
2734 max_delay: Duration::from_secs(5),
2735 jitter: false,
2736 connect_timeout: None,
2737 });
2738 let service = AutoPipelineService::with_factory_and_events(
2739 factory,
2740 AutoPipelineConfig::default(),
2741 reconnect,
2742 events,
2743 )
2744 .await
2745 .unwrap();
2746 let mut surviving_clone = service.clone();
2747 let shutdown = service.shutdown_handle();
2748 assert_eq!(
2749 event_stream.recv().await.unwrap(),
2750 ConnectionEvent::Connected
2751 );
2752
2753 first_closed_rx.await.unwrap();
2754 assert!(matches!(
2755 event_stream.recv().await.unwrap(),
2756 ConnectionEvent::Disconnected {
2757 reason: ConnectionDisconnectReason::ConnectionError { .. }
2758 }
2759 ));
2760 assert_eq!(
2761 event_stream.recv().await.unwrap(),
2762 ConnectionEvent::ReconnectAttempt {
2763 attempt: 1,
2764 delay: Duration::ZERO,
2765 }
2766 );
2767 reconnect_entered.notified().await;
2768 futures::future::poll_fn(|cx| surviving_clone.poll_ready(cx))
2769 .await
2770 .unwrap();
2771
2772 tokio::time::timeout(Duration::from_secs(1), shutdown.shutdown())
2773 .await
2774 .expect("controlled shutdown must cancel a hanging reconnect factory");
2775 assert_eq!(
2776 event_stream.recv().await.unwrap(),
2777 ConnectionEvent::Disconnected {
2778 reason: ConnectionDisconnectReason::Shutdown,
2779 }
2780 );
2781 assert!(matches!(
2782 surviving_clone
2783 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
2784 .await,
2785 Err(RedisError::ConnectionClosed)
2786 ));
2787 assert!(shutdown.is_shutdown_requested());
2788 assert!(shutdown.is_finished());
2789 drop(surviving_clone);
2790 drop(service);
2791 drop(shutdown);
2792 assert_eq!(
2793 event_stream.recv().await.unwrap_err(),
2794 crate::reconnect::ConnectionEventRecvError::Closed
2795 );
2796 assert_eq!(calls.load(Ordering::Acquire), 2);
2797
2798 server.await.unwrap();
2799 }
2800
2801 #[tokio::test]
2802 async fn exhausted_worker_still_publishes_shutdown_on_final_handle() {
2803 use std::sync::atomic::{AtomicUsize, Ordering};
2804 use tokio::sync::oneshot;
2805
2806 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2807 let addr = listener.local_addr().unwrap();
2808 let (first_closed_tx, first_closed_rx) = oneshot::channel();
2809 let server = tokio::spawn(async move {
2810 let (first, _) = listener.accept().await.unwrap();
2811 drop(first);
2812 let _ = first_closed_tx.send(());
2813 });
2814
2815 let calls = Arc::new(AtomicUsize::new(0));
2816 let factory_calls = Arc::clone(&calls);
2817 let factory = move || {
2818 let factory_calls = Arc::clone(&factory_calls);
2819 async move {
2820 if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
2821 return Err(RedisError::ConnectionClosed);
2822 }
2823 let stream = tokio::net::TcpStream::connect(addr)
2824 .await
2825 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2826 Ok(RedisConnection::from_stream(
2827 redis_tower_core::RedisStream::Tcp(stream),
2828 ))
2829 }
2830 };
2831
2832 let events = ConnectionEventBus::new(8);
2833 let mut event_stream = events.subscribe();
2834 let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2835 max_retries: Some(0),
2836 base_delay: Duration::ZERO,
2837 max_delay: Duration::ZERO,
2838 jitter: false,
2839 connect_timeout: None,
2840 });
2841 let service = AutoPipelineService::with_factory_and_events(
2842 factory,
2843 AutoPipelineConfig::default(),
2844 reconnect,
2845 events,
2846 )
2847 .await
2848 .unwrap();
2849 assert_eq!(
2850 event_stream.recv().await.unwrap(),
2851 ConnectionEvent::Connected
2852 );
2853
2854 first_closed_rx.await.unwrap();
2855 assert!(matches!(
2856 event_stream.recv().await.unwrap(),
2857 ConnectionEvent::Disconnected {
2858 reason: ConnectionDisconnectReason::ConnectionError { .. }
2859 }
2860 ));
2861 assert!(matches!(
2862 event_stream.recv().await.unwrap(),
2863 ConnectionEvent::ReconnectAttempt { attempt: 1, .. }
2864 ));
2865 assert!(matches!(
2866 event_stream.recv().await.unwrap(),
2867 ConnectionEvent::ReconnectFailed { attempt: 1, .. }
2868 ));
2869 assert_eq!(
2870 event_stream.recv().await.unwrap(),
2871 ConnectionEvent::ReconnectExhausted { attempts: 1 }
2872 );
2873
2874 tokio::time::timeout(Duration::from_secs(1), service.shutdown())
2875 .await
2876 .expect("shutdown should join an already exhausted worker");
2877 assert_eq!(
2878 event_stream.recv().await.unwrap(),
2879 ConnectionEvent::Disconnected {
2880 reason: ConnectionDisconnectReason::Shutdown,
2881 }
2882 );
2883 assert_eq!(
2884 event_stream.recv().await.unwrap_err(),
2885 crate::reconnect::ConnectionEventRecvError::Closed
2886 );
2887 assert_eq!(calls.load(Ordering::Acquire), 2);
2888
2889 server.await.unwrap();
2890 }
2891
2892 #[test]
2893 fn config_defaults() {
2894 let config = AutoPipelineConfig::default();
2895 assert_eq!(config.max_batch_size, 100);
2896 assert_eq!(config.batch_window, Duration::ZERO);
2897 assert_eq!(config.queue_capacity, 1024);
2898 assert!(
2899 !config.shed_load_on_full,
2900 "back-pressure is the default; load shedding is opt-in"
2901 );
2902 assert!(
2903 config.response_timeout.is_none(),
2904 "no response deadline by default"
2905 );
2906 assert!(
2907 config.metrics_recorder.is_none(),
2908 "no metrics recorder by default"
2909 );
2910 }
2911
2912 #[test]
2913 fn config_custom() {
2914 let config = AutoPipelineConfig {
2915 max_batch_size: 50,
2916 batch_window: Duration::from_micros(500),
2917 queue_capacity: 512,
2918 shed_load_on_full: true,
2919 response_timeout: Some(Duration::from_millis(250)),
2920 metrics_recorder: None,
2921 reconnect_on_readonly: false,
2922 };
2923 assert_eq!(config.max_batch_size, 50);
2924 assert_eq!(config.batch_window, Duration::from_micros(500));
2925 assert_eq!(config.queue_capacity, 512);
2926 assert!(config.shed_load_on_full);
2927 assert_eq!(config.response_timeout, Some(Duration::from_millis(250)));
2928 assert!(!config.reconnect_on_readonly);
2929 }
2930
2931 #[cfg(unix)]
2932 #[tokio::test]
2933 async fn fixed_worker_detects_idle_close_and_closes_health_and_queue() {
2934 let (client, server) = tokio::net::UnixStream::pair().unwrap();
2935 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
2936 let events = ConnectionEventBus::new(4);
2937 let mut event_stream = events.subscribe();
2938 let mut service =
2939 AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
2940 let mut health = service.subscribe_connection_health();
2941
2942 assert!(service.is_connection_healthy());
2943 assert!(*health.borrow());
2944 assert_eq!(
2945 event_stream.recv().await.unwrap(),
2946 ConnectionEvent::Connected
2947 );
2948
2949 drop(server);
2950 let disconnected = tokio::time::timeout(Duration::from_secs(1), event_stream.recv())
2951 .await
2952 .expect("idle close was not detected")
2953 .unwrap();
2954 assert!(matches!(
2955 disconnected,
2956 ConnectionEvent::Disconnected {
2957 reason: ConnectionDisconnectReason::ConnectionError { .. }
2958 }
2959 ));
2960 assert!(
2961 !service.is_connection_healthy(),
2962 "health must be false before Disconnected is observable"
2963 );
2964
2965 health
2966 .changed()
2967 .await
2968 .expect("the final unhealthy snapshot must be observable");
2969 assert!(!*health.borrow_and_update());
2970 assert!(
2971 health.changed().await.is_err(),
2972 "terminal fixed-worker loss must close the health channel"
2973 );
2974 assert!(
2975 futures::future::poll_fn(|cx| service.poll_ready(cx))
2976 .await
2977 .is_err(),
2978 "terminal idle loss must close the request queue"
2979 );
2980
2981 service.shutdown().await;
2982 }
2983
2984 #[cfg(unix)]
2985 #[tokio::test]
2986 async fn pipeline_failure_publishes_unhealthy_before_failing_responder() {
2987 use futures::StreamExt;
2988 use tokio_util::codec::Framed;
2989
2990 let (client, server) = tokio::net::UnixStream::pair().unwrap();
2991 let server = tokio::spawn(async move {
2992 let mut framed = Framed::new(
2993 redis_tower_core::RedisStream::Unix(server),
2994 redis_tower_core::RespCodec::new(),
2995 );
2996 framed
2997 .next()
2998 .await
2999 .expect("client closed before dispatch")
3000 .expect("client sent an invalid request");
3001 });
3004
3005 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3006 let events = ConnectionEventBus::new(4);
3007 let mut event_stream = events.subscribe();
3008 let mut service =
3009 AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
3010 assert_eq!(
3011 event_stream.recv().await.unwrap(),
3012 ConnectionEvent::Connected
3013 );
3014
3015 futures::future::poll_fn(|cx| service.poll_ready(cx))
3016 .await
3017 .unwrap();
3018 assert!(
3019 service
3020 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3021 .await
3022 .is_err()
3023 );
3024 assert!(
3025 !service.is_connection_healthy(),
3026 "the response future observed failure before shared health changed"
3027 );
3028 assert!(matches!(
3029 event_stream.recv().await.unwrap(),
3030 ConnectionEvent::Disconnected {
3031 reason: ConnectionDisconnectReason::ConnectionError { .. }
3032 }
3033 ));
3034
3035 service.shutdown().await;
3036 server.await.unwrap();
3037 }
3038
3039 #[cfg(unix)]
3040 #[tokio::test]
3041 async fn response_timeout_publishes_unhealthy_before_failing_responder() {
3042 use futures::StreamExt;
3043 use tokio_util::codec::Framed;
3044
3045 let (client, server) = tokio::net::UnixStream::pair().unwrap();
3046 let server = tokio::spawn(async move {
3047 let mut framed = Framed::new(
3048 redis_tower_core::RedisStream::Unix(server),
3049 redis_tower_core::RespCodec::new(),
3050 );
3051 framed
3052 .next()
3053 .await
3054 .expect("client closed before dispatch")
3055 .expect("client sent an invalid request");
3056 let _ = framed.next().await;
3059 });
3060
3061 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3062 let events = ConnectionEventBus::new(4);
3063 let mut event_stream = events.subscribe();
3064 let config = AutoPipelineConfig {
3065 response_timeout: Some(Duration::from_millis(10)),
3066 ..AutoPipelineConfig::default()
3067 };
3068 let mut service = AutoPipelineService::new_with_events(conn, config, events);
3069 assert_eq!(
3070 event_stream.recv().await.unwrap(),
3071 ConnectionEvent::Connected
3072 );
3073
3074 futures::future::poll_fn(|cx| service.poll_ready(cx))
3075 .await
3076 .unwrap();
3077 assert!(matches!(
3078 service
3079 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3080 .await,
3081 Err(RedisError::CommandTimeout)
3082 ));
3083 assert!(
3084 !service.is_connection_healthy(),
3085 "the timeout responder was notified before shared health changed"
3086 );
3087 assert_eq!(
3088 event_stream.recv().await.unwrap(),
3089 ConnectionEvent::Disconnected {
3090 reason: ConnectionDisconnectReason::CommandTimeout,
3091 }
3092 );
3093
3094 service.shutdown().await;
3095 server.await.unwrap();
3096 }
3097
3098 #[cfg(unix)]
3099 #[tokio::test]
3100 async fn idle_push_is_routed_without_delaying_the_next_request() {
3101 use futures::{SinkExt, StreamExt};
3102 use tokio_util::codec::Framed;
3103
3104 let (client, server) = tokio::net::UnixStream::pair().unwrap();
3105 let mut conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3106 let mut pushes = conn.subscribe_pushes();
3107 let expected_push = Frame::Push(vec![
3108 Frame::SimpleString(bytes::Bytes::from_static(b"invalidate")),
3109 Frame::Array(Some(vec![Frame::SimpleString(bytes::Bytes::from_static(
3110 b"key",
3111 ))])),
3112 ]);
3113 let server_push = expected_push.clone();
3114 let server = tokio::spawn(async move {
3115 let mut framed = Framed::new(
3116 redis_tower_core::RedisStream::Unix(server),
3117 redis_tower_core::RespCodec::new(),
3118 );
3119 framed.send(server_push).await.unwrap();
3120 framed
3121 .next()
3122 .await
3123 .expect("client closed before request")
3124 .expect("client sent an invalid request");
3125 framed
3126 .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3127 .await
3128 .unwrap();
3129 let _ = framed.next().await;
3132 });
3133
3134 let mut service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
3135 assert_eq!(
3136 tokio::time::timeout(Duration::from_secs(1), pushes.recv())
3137 .await
3138 .expect("idle worker did not route the push")
3139 .unwrap(),
3140 expected_push
3141 );
3142 futures::future::poll_fn(|cx| service.poll_ready(cx))
3143 .await
3144 .unwrap();
3145 assert_eq!(
3146 service
3147 .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3148 .await
3149 .unwrap(),
3150 Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3151 );
3152 assert!(service.is_connection_healthy());
3153
3154 service.shutdown().await;
3155 server.await.unwrap();
3156 }
3157
3158 #[cfg(unix)]
3159 #[tokio::test]
3160 async fn cancelled_single_and_multi_requests_are_pruned_before_wire() {
3161 use tokio::io::AsyncReadExt;
3162
3163 let (client, mut server) = tokio::net::UnixStream::pair().unwrap();
3164 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3165 let batch_window = Duration::from_millis(200);
3166 let mut service = AutoPipelineService::new(
3167 conn,
3168 AutoPipelineConfig {
3169 batch_window,
3170 ..AutoPipelineConfig::default()
3171 },
3172 );
3173
3174 futures::future::poll_fn(|cx| service.poll_ready(cx))
3175 .await
3176 .unwrap();
3177 let single = service.call(Frame::SimpleString(bytes::Bytes::from_static(
3178 b"CANCELLED-SINGLE",
3179 )));
3180 assert!(
3181 tokio::time::timeout(Duration::from_millis(25), single)
3182 .await
3183 .is_err(),
3184 "single request unexpectedly completed without a server response"
3185 );
3186
3187 let multi = service.call_pipeline(vec![
3188 Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-1")),
3189 Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-2")),
3190 ]);
3191 assert!(
3192 tokio::time::timeout(Duration::from_millis(25), multi)
3193 .await
3194 .is_err(),
3195 "multi request unexpectedly completed without a server response"
3196 );
3197
3198 let mut bytes = [0u8; 256];
3202 assert!(
3203 tokio::time::timeout(
3204 batch_window + Duration::from_millis(100),
3205 server.read(&mut bytes)
3206 )
3207 .await
3208 .is_err(),
3209 "a cancelled queued request reached the Redis socket"
3210 );
3211
3212 service.shutdown().await;
3213 }
3214
3215 #[tokio::test]
3216 async fn lost_non_idempotent_reply_is_unknown_and_not_replayed_after_reconnect() {
3217 use futures::{SinkExt, StreamExt};
3218 use std::sync::atomic::{AtomicUsize, Ordering};
3219 use tokio_util::codec::Framed;
3220
3221 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3222 let addr = listener.local_addr().unwrap();
3223 let executions = Arc::new(AtomicUsize::new(0));
3224 let server_executions = Arc::clone(&executions);
3225 let server = tokio::spawn(async move {
3226 let (first, _) = listener.accept().await.unwrap();
3227 let mut first = Framed::new(
3228 redis_tower_core::RedisStream::Tcp(first),
3229 redis_tower_core::RespCodec::new(),
3230 );
3231 let request = first
3232 .next()
3233 .await
3234 .expect("client closed before dispatch")
3235 .expect("client sent an invalid request");
3236 assert_eq!(
3237 request,
3238 Frame::SimpleString(bytes::Bytes::from_static(b"INCR-NON-IDEMPOTENT"))
3239 );
3240 server_executions.fetch_add(1, Ordering::SeqCst);
3241 drop(first);
3244
3245 let (second, _) = listener.accept().await.unwrap();
3246 let mut second = Framed::new(
3247 redis_tower_core::RedisStream::Tcp(second),
3248 redis_tower_core::RespCodec::new(),
3249 );
3250 let request = second
3251 .next()
3252 .await
3253 .expect("client closed before post-reconnect probe")
3254 .expect("client sent an invalid post-reconnect request");
3255 assert_eq!(
3256 request,
3257 Frame::SimpleString(bytes::Bytes::from_static(b"READ-EXECUTION-COUNT")),
3258 "the lost-reply INCR was replayed on the replacement connection"
3259 );
3260 second
3261 .send(Frame::Integer(
3262 server_executions.load(Ordering::SeqCst) as i64
3263 ))
3264 .await
3265 .unwrap();
3266 let _ = second.next().await;
3267 });
3268
3269 let factory = move || async move {
3270 let stream = tokio::net::TcpStream::connect(addr)
3271 .await
3272 .map_err(|error| RedisError::connection(addr.to_string(), error))?;
3273 Ok(RedisConnection::from_stream(
3274 redis_tower_core::RedisStream::Tcp(stream),
3275 ))
3276 };
3277 let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
3278 max_retries: Some(3),
3279 base_delay: Duration::ZERO,
3280 max_delay: Duration::ZERO,
3281 jitter: false,
3282 connect_timeout: None,
3283 });
3284 let mut service =
3285 AutoPipelineService::with_factory(factory, AutoPipelineConfig::default(), reconnect)
3286 .await
3287 .unwrap();
3288
3289 futures::future::poll_fn(|cx| service.poll_ready(cx))
3290 .await
3291 .unwrap();
3292 let lost = tokio::time::timeout(
3293 Duration::from_secs(1),
3294 service.call(Frame::SimpleString(bytes::Bytes::from_static(
3295 b"INCR-NON-IDEMPOTENT",
3296 ))),
3297 )
3298 .await
3299 .expect("lost-reply command did not complete");
3300 assert!(lost.is_err(), "a dropped reply was reported as success");
3301
3302 futures::future::poll_fn(|cx| service.poll_ready(cx))
3303 .await
3304 .unwrap();
3305 let count = tokio::time::timeout(
3306 Duration::from_secs(1),
3307 service.call(Frame::SimpleString(bytes::Bytes::from_static(
3308 b"READ-EXECUTION-COUNT",
3309 ))),
3310 )
3311 .await
3312 .expect("post-reconnect probe timed out")
3313 .unwrap();
3314 assert_eq!(count, Frame::Integer(1));
3315 assert_eq!(executions.load(Ordering::SeqCst), 1);
3316
3317 service.shutdown().await;
3318 server.await.unwrap();
3319 }
3320
3321 #[test]
3322 fn is_readonly_frame_detects_readonly_errors_only() {
3323 use bytes::Bytes;
3324 assert!(is_readonly_frame(&Frame::Error(Bytes::from(
3325 "READONLY You can't write against a read only replica."
3326 ))));
3327 assert!(is_readonly_frame(&Frame::Error(Bytes::from(
3329 "readonly nope"
3330 ))));
3331 assert!(!is_readonly_frame(&Frame::Error(Bytes::from(
3333 "WRONGTYPE Operation against a key"
3334 ))));
3335 assert!(!is_readonly_frame(&Frame::Error(Bytes::from("READ"))));
3336 assert!(!is_readonly_frame(&Frame::SimpleString(Bytes::from("OK"))));
3337 }
3338
3339 fn make_test_svc(
3340 tx: mpsc::Sender<WorkerRequest>,
3341 handle: tokio::task::JoinHandle<()>,
3342 shed_load: bool,
3343 ) -> AutoPipelineService {
3344 let (shutdown_tx, _shutdown_rx) = watch::channel(false);
3345 let (_connection_health_tx, connection_health) = watch::channel(true);
3346 let control = Arc::new(WorkerControl::new(shutdown_tx, None));
3347 AutoPipelineService {
3348 poll_tx: PollSender::new(tx.clone()),
3349 tx,
3350 shed_load,
3351 shutdown_notified: Box::pin(control.shutdown_notified()),
3352 lease: WorkerLease { control },
3353 connection_health,
3354 worker: Arc::new(WorkerHandle::new(handle)),
3355 }
3356 }
3357
3358 #[tokio::test]
3359 async fn closed_channel_error_is_retryable() {
3360 let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3363 drop(rx);
3364 let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3365
3366 let frame = Frame::SimpleString(b"PING"[..].into());
3367 let err = svc.call(frame).await.unwrap_err();
3368 assert!(err.is_retryable());
3369 }
3370
3371 #[tokio::test]
3372 async fn closed_channel_returns_error() {
3373 let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3375 drop(rx); let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3377
3378 let ready = futures::future::poll_fn(|cx| svc.poll_ready(cx)).await;
3380 assert!(ready.is_err());
3381
3382 let frame = Frame::SimpleString(b"PING"[..].into());
3384 let result = svc.call(frame).await;
3385 assert!(result.is_err());
3386 }
3387
3388 #[tokio::test]
3389 async fn shutdown_drains_an_unpolled_shed_load_call() {
3390 use futures::{SinkExt, StreamExt};
3391 use tokio_util::codec::Framed;
3392
3393 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3394 let addr = listener.local_addr().unwrap();
3395 let request = Frame::Array(Some(vec![Frame::BulkString(Some(
3396 bytes::Bytes::from_static(b"PING"),
3397 ))]));
3398 let expected_request = request.clone();
3399 let server = tokio::spawn(async move {
3400 let (stream, _) = listener.accept().await.unwrap();
3401 let mut framed = Framed::new(
3402 redis_tower_core::RedisStream::Tcp(stream),
3403 redis_tower_core::RespCodec::new(),
3404 );
3405 assert_eq!(framed.next().await.unwrap().unwrap(), expected_request);
3406 framed
3407 .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3408 .await
3409 .unwrap();
3410 });
3411
3412 let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3413 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3414 let events = ConnectionEventBus::new(4);
3415 let mut event_stream = events.subscribe();
3416 let config = AutoPipelineConfig {
3417 batch_window: Duration::from_secs(30),
3420 shed_load_on_full: true,
3421 ..AutoPipelineConfig::default()
3422 };
3423 let mut service = AutoPipelineService::new_with_events(conn, config, events);
3424 assert_eq!(
3425 event_stream.recv().await.unwrap(),
3426 ConnectionEvent::Connected
3427 );
3428
3429 let response = service.call(request);
3433 tokio::time::timeout(Duration::from_secs(1), service.shutdown())
3434 .await
3435 .expect("an unpolled shed-load call must not keep shutdown alive");
3436 assert_eq!(
3437 response.await.unwrap(),
3438 Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3439 );
3440 assert_eq!(
3441 event_stream.recv().await.unwrap(),
3442 ConnectionEvent::Disconnected {
3443 reason: ConnectionDisconnectReason::Shutdown,
3444 }
3445 );
3446 assert_eq!(
3447 event_stream.recv().await.unwrap_err(),
3448 crate::reconnect::ConnectionEventRecvError::Closed
3449 );
3450
3451 server.await.unwrap();
3452 }
3453
3454 #[tokio::test]
3455 async fn shutdown_handle_drains_trait_erased_clones_and_is_idempotent() {
3456 use futures::{SinkExt, StreamExt};
3457 use tokio::sync::oneshot;
3458 use tokio_util::codec::Framed;
3459 use tower::util::BoxCloneService;
3460
3461 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3462 let addr = listener.local_addr().unwrap();
3463 let request = Frame::SimpleString(bytes::Bytes::from_static(b"PING"));
3464 let expected_request = request.clone();
3465 let (request_seen_tx, request_seen_rx) = oneshot::channel();
3466 let (send_reply_tx, send_reply_rx) = oneshot::channel();
3467 let server = tokio::spawn(async move {
3468 let (stream, _) = listener.accept().await.unwrap();
3469 let mut framed = Framed::new(
3470 redis_tower_core::RedisStream::Tcp(stream),
3471 redis_tower_core::RespCodec::new(),
3472 );
3473 assert_eq!(framed.next().await.unwrap().unwrap(), expected_request);
3474 let _ = request_seen_tx.send(());
3475 send_reply_rx.await.unwrap();
3476 framed
3477 .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3478 .await
3479 .unwrap();
3480 assert!(framed.next().await.is_none());
3481 });
3482
3483 let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3484 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3485 let events = ConnectionEventBus::new(4);
3486 let mut event_stream = events.subscribe();
3487 let service = AutoPipelineService::new_with_events(
3488 conn,
3489 AutoPipelineConfig {
3490 batch_window: Duration::from_secs(30),
3491 ..AutoPipelineConfig::default()
3492 },
3493 events,
3494 );
3495 assert_eq!(
3496 event_stream.recv().await.unwrap(),
3497 ConnectionEvent::Connected
3498 );
3499
3500 let shutdown = service.shutdown_handle();
3501 let mut erased: BoxCloneService<Frame, Frame, RedisError> = BoxCloneService::new(service);
3502 let mut surviving_clone = erased.clone();
3503 let mut idle_reserved_clone = erased.clone();
3504 futures::future::poll_fn(|cx| erased.poll_ready(cx))
3505 .await
3506 .unwrap();
3507 let response = erased.call(request);
3508 futures::future::poll_fn(|cx| idle_reserved_clone.poll_ready(cx))
3509 .await
3510 .unwrap();
3511
3512 let first_shutdown = shutdown.clone();
3513 let second_shutdown = shutdown.clone();
3514 let shutdown_task = tokio::spawn(async move {
3515 tokio::join!(first_shutdown.shutdown(), second_shutdown.shutdown());
3516 });
3517 tokio::time::timeout(Duration::from_secs(1), request_seen_rx)
3518 .await
3519 .expect("controlled shutdown did not drain the accepted request")
3520 .unwrap();
3521 send_reply_tx.send(()).unwrap();
3522 tokio::time::timeout(Duration::from_secs(1), shutdown_task)
3523 .await
3524 .expect("concurrent shutdown handles did not join the worker")
3525 .unwrap();
3526
3527 assert_eq!(
3528 response.await.unwrap(),
3529 Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3530 );
3531 assert!(shutdown.is_shutdown_requested());
3532 assert!(shutdown.is_finished());
3533 assert!(
3534 futures::future::poll_fn(|cx| surviving_clone.poll_ready(cx))
3535 .await
3536 .is_err()
3537 );
3538 assert!(matches!(
3539 surviving_clone.call(Frame::Integer(1)).await,
3540 Err(RedisError::ConnectionClosed)
3541 ));
3542 assert!(matches!(
3543 idle_reserved_clone.call(Frame::Integer(2)).await,
3544 Err(RedisError::ConnectionClosed)
3545 ));
3546 assert_eq!(
3547 event_stream.recv().await.unwrap(),
3548 ConnectionEvent::Disconnected {
3549 reason: ConnectionDisconnectReason::Shutdown,
3550 }
3551 );
3552
3553 drop(erased);
3554 drop(surviving_clone);
3555 drop(idle_reserved_clone);
3556 drop(shutdown);
3557 assert_eq!(
3558 event_stream.recv().await.unwrap_err(),
3559 crate::reconnect::ConnectionEventRecvError::Closed
3560 );
3561 server.await.unwrap();
3562 }
3563
3564 #[tokio::test]
3565 async fn shutdown_handle_wakes_blocked_backpressure_before_drain_completes() {
3566 use futures::{SinkExt, StreamExt};
3567 use tokio::sync::oneshot;
3568 use tokio_util::codec::Framed;
3569
3570 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3571 let addr = listener.local_addr().unwrap();
3572 let first_request = Frame::SimpleString(bytes::Bytes::from_static(b"FIRST"));
3573 let second_request = Frame::SimpleString(bytes::Bytes::from_static(b"SECOND"));
3574 let (first_seen_tx, first_seen_rx) = oneshot::channel();
3575 let (release_first_tx, release_first_rx) = oneshot::channel();
3576 let server = tokio::spawn({
3577 let first_request = first_request.clone();
3578 let second_request = second_request.clone();
3579 async move {
3580 let (stream, _) = listener.accept().await.unwrap();
3581 let mut framed = Framed::new(
3582 redis_tower_core::RedisStream::Tcp(stream),
3583 redis_tower_core::RespCodec::new(),
3584 );
3585 assert_eq!(framed.next().await.unwrap().unwrap(), first_request);
3586 let _ = first_seen_tx.send(());
3587 release_first_rx.await.unwrap();
3588 framed.send(Frame::Integer(1)).await.unwrap();
3589 assert_eq!(framed.next().await.unwrap().unwrap(), second_request);
3590 framed.send(Frame::Integer(2)).await.unwrap();
3591 assert!(framed.next().await.is_none());
3592 }
3593 });
3594
3595 let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3596 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3597 let mut first = AutoPipelineService::new(
3598 conn,
3599 AutoPipelineConfig {
3600 queue_capacity: 1,
3601 max_batch_size: 1,
3602 batch_window: Duration::ZERO,
3603 ..AutoPipelineConfig::default()
3604 },
3605 );
3606 let shutdown = first.shutdown_handle();
3607
3608 futures::future::poll_fn(|cx| first.poll_ready(cx))
3609 .await
3610 .unwrap();
3611 let first_response = first.call(first_request);
3612 first_seen_rx.await.unwrap();
3613
3614 let mut second = first.clone();
3615 futures::future::poll_fn(|cx| second.poll_ready(cx))
3616 .await
3617 .unwrap();
3618 let second_response = second.call(second_request);
3619
3620 let mut pending_ready_service = first.clone();
3621 let (ready_pending_tx, ready_pending_rx) = oneshot::channel();
3622 let pending_ready_task = tokio::spawn(async move {
3623 let mut ready_pending_tx = Some(ready_pending_tx);
3624 futures::future::poll_fn(|cx| {
3625 let poll = pending_ready_service.poll_ready(cx);
3626 if poll.is_pending()
3627 && let Some(ready_pending_tx) = ready_pending_tx.take()
3628 {
3629 let _ = ready_pending_tx.send(());
3630 }
3631 poll
3632 })
3633 .await
3634 });
3635
3636 let mut pending_pipeline_service = first.clone();
3637 let (pipeline_pending_tx, pipeline_pending_rx) = oneshot::channel();
3638 let pending_pipeline_task = tokio::spawn(async move {
3639 let pending_pipeline = pending_pipeline_service.call_pipeline(vec![Frame::Integer(3)]);
3640 tokio::pin!(pending_pipeline);
3641 let mut pipeline_pending_tx = Some(pipeline_pending_tx);
3642 futures::future::poll_fn(|cx| {
3643 let poll = pending_pipeline.as_mut().poll(cx);
3644 if poll.is_pending()
3645 && let Some(pipeline_pending_tx) = pipeline_pending_tx.take()
3646 {
3647 let _ = pipeline_pending_tx.send(());
3648 }
3649 poll
3650 })
3651 .await
3652 });
3653
3654 ready_pending_rx
3655 .await
3656 .expect("poll_ready completed before the queue was released");
3657 pipeline_pending_rx
3658 .await
3659 .expect("call_pipeline completed before the queue was released");
3660
3661 let shutdown_task = tokio::spawn({
3662 let shutdown = shutdown.clone();
3663 async move { shutdown.shutdown().await }
3664 });
3665 while !shutdown.is_shutdown_requested() {
3666 tokio::task::yield_now().await;
3667 }
3668
3669 assert!(matches!(
3670 tokio::time::timeout(Duration::from_secs(1), pending_ready_task)
3671 .await
3672 .expect("pending poll_ready was not woken by shutdown")
3673 .unwrap(),
3674 Err(RedisError::ConnectionClosed)
3675 ));
3676 assert!(matches!(
3677 tokio::time::timeout(Duration::from_secs(1), pending_pipeline_task)
3678 .await
3679 .expect("pending call_pipeline was not woken by shutdown")
3680 .unwrap(),
3681 Err(RedisError::ConnectionClosed)
3682 ));
3683
3684 let mut new_pipeline_service = first.clone();
3685 assert!(matches!(
3686 tokio::time::timeout(
3687 Duration::from_secs(1),
3688 new_pipeline_service.call_pipeline(vec![Frame::Integer(4)]),
3689 )
3690 .await
3691 .expect("a new call_pipeline blocked after shutdown"),
3692 Err(RedisError::ConnectionClosed)
3693 ));
3694 assert!(
3695 !shutdown_task.is_finished(),
3696 "shutdown returned before accepted work drained"
3697 );
3698
3699 release_first_tx.send(()).unwrap();
3700 tokio::time::timeout(Duration::from_secs(1), shutdown_task)
3701 .await
3702 .expect("shutdown did not finish after accepted work drained")
3703 .unwrap();
3704 assert_eq!(first_response.await.unwrap(), Frame::Integer(1));
3705 assert_eq!(second_response.await.unwrap(), Frame::Integer(2));
3706
3707 drop(first);
3708 drop(second);
3709 drop(new_pipeline_service);
3710 drop(shutdown);
3711 server.await.unwrap();
3712 }
3713
3714 #[tokio::test]
3715 async fn final_shutdown_closes_an_idle_worker_with_a_retained_sender() {
3716 use tokio::io::AsyncReadExt;
3717
3718 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3719 let addr = listener.local_addr().unwrap();
3720 let server = tokio::spawn(async move {
3721 let (mut stream, _) = listener.accept().await.unwrap();
3722 let mut byte = [0_u8; 1];
3723 assert_eq!(stream.read(&mut byte).await.unwrap(), 0);
3724 });
3725
3726 let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3727 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3728 let service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
3729
3730 let retained_sender = service.tx.clone();
3734 tokio::time::timeout(Duration::from_secs(1), service.shutdown())
3735 .await
3736 .expect("final shutdown must wake an idle worker");
3737 assert!(retained_sender.is_closed());
3738 drop(retained_sender);
3739
3740 tokio::time::timeout(Duration::from_secs(1), server)
3741 .await
3742 .expect("worker shutdown must close its Redis connection")
3743 .unwrap();
3744 }
3745
3746 #[tokio::test]
3747 async fn shutdown_last_clone_awaits_worker() {
3748 let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3751 let handle = tokio::spawn(async move {
3752 let mut rx = rx;
3754 while rx.recv().await.is_some() {}
3755 });
3756 let svc = make_test_svc(tx, handle, false);
3757
3758 svc.shutdown().await;
3761 }
3763
3764 #[tokio::test]
3765 async fn shutdown_publishes_one_intentional_disconnect() {
3766 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3767 let addr = listener.local_addr().unwrap();
3768 let server = tokio::spawn(async move {
3769 let (stream, _) = listener.accept().await.unwrap();
3770 let _stream = stream;
3771 futures::future::pending::<()>().await;
3772 });
3773 let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3774 let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3775 let events = ConnectionEventBus::new(4);
3776 let mut event_stream = events.subscribe();
3777 let service =
3778 AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
3779 assert_eq!(
3780 event_stream.recv().await.unwrap(),
3781 ConnectionEvent::Connected
3782 );
3783
3784 service.shutdown().await;
3785 assert_eq!(
3786 event_stream.recv().await.unwrap(),
3787 ConnectionEvent::Disconnected {
3788 reason: ConnectionDisconnectReason::Shutdown,
3789 }
3790 );
3791
3792 server.abort();
3793 }
3794
3795 #[tokio::test]
3796 async fn shutdown_non_last_clone_returns_immediately() {
3797 let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3800 let handle = tokio::spawn(futures::future::pending::<()>());
3802 let svc = make_test_svc(tx, handle, false);
3803
3804 let _clone = svc.clone();
3806
3807 svc.shutdown().await;
3809
3810 }
3813
3814 #[tokio::test]
3815 async fn is_alive_tracks_worker_exit() {
3816 let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3818 let alive = make_test_svc(tx, tokio::spawn(futures::future::pending::<()>()), false);
3819 assert!(alive.is_alive());
3820
3821 let (tx2, _rx2) = mpsc::channel::<WorkerRequest>(1);
3823 let dead = make_test_svc(tx2, tokio::spawn(async {}), false);
3824 for _ in 0..10 {
3826 if !dead.is_alive() {
3827 break;
3828 }
3829 tokio::task::yield_now().await;
3830 }
3831 assert!(!dead.is_alive());
3832 }
3833
3834 #[tokio::test]
3835 async fn queue_full_returns_queue_full_error() {
3836 let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3838 let (dummy_tx, _dummy_rx) = oneshot::channel();
3840 tx.try_send(WorkerRequest::Single {
3841 frame: Frame::SimpleString(b"PING"[..].into()),
3842 response_tx: dummy_tx,
3843 })
3844 .unwrap();
3845
3846 let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3847
3848 let frame = Frame::SimpleString(b"PING"[..].into());
3849 let err = svc.call(frame).await.unwrap_err();
3850 assert!(
3851 matches!(err, RedisError::QueueFull),
3852 "expected QueueFull, got {err:?}"
3853 );
3854 }
3855
3856 #[tokio::test]
3857 async fn backpressure_poll_ready_pends_when_full() {
3858 let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3861 let (dummy_tx, _dummy_rx) = oneshot::channel();
3863 tx.try_send(WorkerRequest::Single {
3864 frame: Frame::SimpleString(b"PING"[..].into()),
3865 response_tx: dummy_tx,
3866 })
3867 .unwrap();
3868
3869 let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3870 let pending = std::future::poll_fn(|cx| Poll::Ready(svc.poll_ready(cx).is_pending())).await;
3871 assert!(
3872 pending,
3873 "poll_ready must pend (back-pressure) when the queue is full"
3874 );
3875 }
3876
3877 #[tokio::test]
3878 async fn reserved_pipeline_consumes_poll_ready_slot_as_one_request() {
3879 let (tx, mut rx) = mpsc::channel::<WorkerRequest>(1);
3880 let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3881
3882 futures::future::poll_fn(|cx| svc.poll_ready(cx))
3883 .await
3884 .unwrap();
3885
3886 let frames = vec![
3887 Frame::SimpleString(b"CLIENT CACHING YES"[..].into()),
3888 Frame::SimpleString(b"GET key"[..].into()),
3889 ];
3890 let response = svc.call_reserved_pipeline(frames.clone());
3891
3892 let request = rx.recv().await.expect("reserved request should be queued");
3893 match request {
3894 WorkerRequest::Multi {
3895 frames: actual,
3896 response_tx,
3897 } => {
3898 assert_eq!(actual, frames);
3899 response_tx
3900 .send(Ok(vec![
3901 Frame::SimpleString(b"OK"[..].into()),
3902 Frame::BulkString(Some(b"value"[..].into())),
3903 ]))
3904 .unwrap();
3905 }
3906 WorkerRequest::Single { .. } => panic!("reserved pipeline was split into a single"),
3907 }
3908
3909 assert_eq!(response.await.unwrap().len(), 2);
3910 }
3911
3912 #[tokio::test]
3913 async fn local_response_can_release_poll_ready_reservation() {
3914 let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3915 let mut svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
3916
3917 assert_eq!(tx.capacity(), 1);
3918 futures::future::poll_fn(|cx| svc.poll_ready(cx))
3919 .await
3920 .unwrap();
3921 assert_eq!(tx.capacity(), 0, "poll_ready should reserve capacity");
3922
3923 svc.release_reservation();
3924 assert_eq!(
3925 tx.capacity(),
3926 1,
3927 "a locally satisfied request must restore queue capacity"
3928 );
3929 }
3930
3931 #[tokio::test]
3932 async fn backpressure_poll_ready_errors_when_closed() {
3933 let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3936 drop(rx);
3937 let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3938 let ready = std::future::poll_fn(|cx| svc.poll_ready(cx)).await;
3939 let err = ready.unwrap_err();
3940 assert!(
3941 err.is_retryable(),
3942 "closed-channel error should be retryable"
3943 );
3944 }
3945
3946 #[tokio::test]
3947 async fn queue_full_not_retryable() {
3948 assert!(!RedisError::QueueFull.is_retryable());
3949 }
3950
3951 #[tokio::test]
3952 async fn queue_full_not_connection_error() {
3953 assert!(!RedisError::QueueFull.is_connection_error());
3954 }
3955
3956 #[test]
3957 fn config_queue_capacity_default() {
3958 let config = AutoPipelineConfig::default();
3959 assert_eq!(config.queue_capacity, 1024);
3960 }
3961
3962 #[tokio::test]
3963 async fn queue_depth_zero_when_empty() {
3964 let (tx, _rx) = mpsc::channel::<WorkerRequest>(64);
3966 let svc = make_test_svc(tx, tokio::spawn(async {}), false);
3967 assert_eq!(svc.queue_depth(), 0);
3968 }
3969
3970 #[tokio::test]
3971 async fn queue_depth_increases_with_pending_requests() {
3972 let (tx, _rx) = mpsc::channel::<WorkerRequest>(10);
3974 let svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
3975
3976 assert_eq!(svc.queue_depth(), 0);
3977
3978 let (dummy_tx, _dummy_rx) = oneshot::channel();
3980 tx.try_send(WorkerRequest::Single {
3981 frame: Frame::SimpleString(b"PING"[..].into()),
3982 response_tx: dummy_tx,
3983 })
3984 .unwrap();
3985
3986 assert_eq!(svc.queue_depth(), 1);
3987 }
3988}