Skip to main content

redis_tower/
auto_pipeline.rs

1//! Auto-pipelining middleware for transparent batching of concurrent requests.
2//!
3//! When multiple tasks issue commands concurrently, instead of sending them
4//! one at a time, [`AutoPipelineService`] collects them into a batch and
5//! sends them as a single Redis pipeline. Each caller gets back their
6//! individual response.
7//!
8//! # Example
9//!
10//! ```no_run
11//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
12//! use redis_tower::{AutoPipelineConfig, AutoPipelineService, CommandAdapter, RedisConnection};
13//! use redis_tower::commands::*;
14//! use tower::Service;
15//!
16//! let conn = RedisConnection::connect("127.0.0.1:6379").await?;
17//! let pipeline = AutoPipelineService::new(conn, AutoPipelineConfig::default());
18//! let shutdown = pipeline.shutdown_handle();
19//! let mut svc = CommandAdapter::new(pipeline);
20//! let value: Option<bytes::Bytes> = svc.call(Get::new("key")).await?;
21//! # let _ = value;
22//! drop(svc);
23//! shutdown.shutdown().await;
24//! # Ok(())
25//! # }
26//! ```
27
28use std::future::Future;
29use std::pin::Pin;
30use std::sync::{Arc, Mutex};
31use std::task::{Context, Poll};
32use std::time::{Duration, Instant};
33
34use redis_tower_core::{Frame, ReceivedPushFrame, RedisConnection, RedisError};
35use tokio::sync::{Mutex as AsyncMutex, Notify, futures::OwnedNotified, mpsc, oneshot, watch};
36use tokio_util::sync::PollSender;
37use tower_service::Service;
38use tracing::warn;
39
40use crate::maintenance::{
41    MaintenanceControl, MaintenanceListenerHandle, MaintenanceState, ParsedMaintenanceNotification,
42    next_listener_id, parse_maintenance_push, register_connection,
43};
44use crate::metrics_layer::MetricsRecorder;
45use crate::reconnect::{
46    ConnectionDisconnectReason, ConnectionEvent, ConnectionEventBus, ConnectionFactory,
47    ReconnectConfig, connect_with_timeout,
48};
49
50/// Configuration for the auto-pipelining service.
51#[derive(Clone)]
52pub struct AutoPipelineConfig {
53    /// Maximum commands to batch before sending. Default: 100.
54    pub max_batch_size: usize,
55    /// Time to wait for more commands after draining the immediate queue.
56    ///
57    /// Default: 0 (no wait -- flushes immediately, only batches requests
58    /// that arrive concurrently). Set to 1-2ms for write-heavy workloads
59    /// where batching reduces round-trips.
60    pub batch_window: Duration,
61    /// Capacity of the internal command queue (number of pending requests
62    /// that can be buffered).
63    ///
64    /// When the queue is full, the default behavior is back-pressure: a new
65    /// request awaits a free slot (its `poll_ready` returns `Pending`). Set
66    /// [`shed_load_on_full`](Self::shed_load_on_full) to fail fast with
67    /// `RedisError::QueueFull` instead.
68    ///
69    /// Default: 1024.
70    pub queue_capacity: usize,
71    /// Fail fast instead of applying back-pressure when the queue is full.
72    ///
73    /// When `false` (default), a caller awaits a free slot before its request
74    /// is accepted -- real back-pressure that paces producers to the worker's
75    /// drain rate. When `true`, a full queue makes the call return
76    /// `RedisError::QueueFull` immediately (load shedding), which suits callers
77    /// that prefer to reject rather than wait.
78    ///
79    /// Default: `false`.
80    pub shed_load_on_full: bool,
81    /// Maximum time to wait for a batch's responses before treating the
82    /// connection as failed.
83    ///
84    /// `None` (default) means no response deadline: a hung or black-holed node
85    /// can stall this worker's whole queue until OS TCP keepalive eventually
86    /// fires (minutes). Set a value so a stuck node is detected promptly -- the
87    /// in-flight batch fails with [`RedisError::CommandTimeout`] and the worker
88    /// discards the connection (factory-backed clients then reconnect with
89    /// backoff, so a new connection serves subsequent requests).
90    ///
91    /// The deadline covers a whole pipelined batch's round-trip, so size it
92    /// above your slowest legitimate command (a long `BLPOP`/`WAIT`/`DEBUG
93    /// SLEEP` will trip it).
94    ///
95    /// Default: `None`.
96    pub response_timeout: Option<Duration>,
97    /// Optional metrics recorder for worker-level observability.
98    ///
99    /// When set, the background worker calls
100    /// [`MetricsRecorder::pipeline_flushed`] after each batch flush, reporting
101    /// how many frames went out together. This is the one signal only the
102    /// worker can see -- a histogram of it shows whether auto-pipelining is
103    /// actually batching (`> 1`) or every caller flushes alone (`== 1`).
104    ///
105    /// For per-command latency/error metrics and tracing spans, wrap the
106    /// client in [`MetricsLayer`](crate::MetricsLayer) /
107    /// [`TracingLayer`](crate::TracingLayer) via
108    /// [`MultiplexedClient::from_layered`](crate::MultiplexedClient::from_layered) --
109    /// those compose at the `Service<Frame>` layer where per-command timing is
110    /// available.
111    ///
112    /// Default: `None`.
113    pub metrics_recorder: Option<Arc<dyn MetricsRecorder>>,
114    /// Treat a `READONLY` reply as a signal that the connection points at a
115    /// replica and the worker should reconnect via its factory.
116    ///
117    /// Off by default. A factory-backed Sentinel client enables this: when a
118    /// master is demoted to a replica (`REPLICAOF`) with TCP intact, writes
119    /// come back `READONLY` rather than as a connection error, so without this
120    /// the worker would keep serving the demoted node forever. With it on, the
121    /// caller still receives the `READONLY` error for the current command and
122    /// the worker reconnects (re-querying Sentinel) so the next batch lands on
123    /// the new master. Standalone and cluster clients leave this off.
124    ///
125    /// Default: `false`.
126    pub reconnect_on_readonly: bool,
127}
128
129impl std::fmt::Debug for AutoPipelineConfig {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        f.debug_struct("AutoPipelineConfig")
132            .field("max_batch_size", &self.max_batch_size)
133            .field("batch_window", &self.batch_window)
134            .field("queue_capacity", &self.queue_capacity)
135            .field("shed_load_on_full", &self.shed_load_on_full)
136            .field("response_timeout", &self.response_timeout)
137            .field(
138                "metrics_recorder",
139                &self.metrics_recorder.as_ref().map(|_| "<recorder>"),
140            )
141            .field("reconnect_on_readonly", &self.reconnect_on_readonly)
142            .finish()
143    }
144}
145
146impl Default for AutoPipelineConfig {
147    fn default() -> Self {
148        Self {
149            max_batch_size: 100,
150            batch_window: Duration::ZERO,
151            queue_capacity: 1024,
152            shed_load_on_full: false,
153            response_timeout: None,
154            metrics_recorder: None,
155            reconnect_on_readonly: false,
156        }
157    }
158}
159
160/// Reconnect policy for a factory-backed [`AutoPipelineService`].
161///
162/// Applies only when the service is constructed via
163/// [`AutoPipelineService::with_factory`] or
164/// [`AutoPipelineService::with_lazy_factory`]. The plain
165/// [`AutoPipelineService::new`] path owns a single pre-built connection and
166/// does not reconnect.
167#[derive(Debug, Clone, Default)]
168pub struct AutoPipelineReconnectConfig {
169    /// Backoff parameters for reconnection attempts.
170    ///
171    /// Its [`ReconnectConfig::connect_timeout`] also bounds the initial
172    /// factory call made by eager and lazy `with_factory` constructors.
173    pub reconnect: ReconnectConfig,
174}
175
176impl AutoPipelineReconnectConfig {
177    /// Create a new reconnect config with the given backoff parameters.
178    pub fn new(reconnect: ReconnectConfig) -> Self {
179        Self { reconnect }
180    }
181}
182
183/// Source of the connection owned by the background worker.
184enum ConnSource {
185    /// Fixed pre-built connection: no reconnect on failure.
186    Fixed,
187    /// Factory-backed connection: rebuild on failure using the factory.
188    Factory {
189        factory: Arc<dyn ConnectionFactory>,
190        reconnect: ReconnectConfig,
191    },
192}
193
194struct MaintenanceWorker {
195    state: MaintenanceState,
196    control_rx: mpsc::UnboundedReceiver<MaintenanceControl>,
197    push_rx: tokio::sync::broadcast::Receiver<ReceivedPushFrame>,
198}
199
200/// Why the pipeline worker discarded its current connection.
201///
202/// The owned Redis error is kept until event publication so formatting it is
203/// skipped entirely when lifecycle events have no subscribers.
204enum PipelineFailure {
205    Connection(RedisError),
206    CommandTimeout,
207    ReadOnly,
208}
209
210impl PipelineFailure {
211    fn event_reason(&self) -> ConnectionDisconnectReason {
212        match self {
213            Self::Connection(error) => ConnectionDisconnectReason::ConnectionError {
214                error: Arc::from(error.to_string()),
215            },
216            Self::CommandTimeout => ConnectionDisconnectReason::CommandTimeout,
217            Self::ReadOnly => ConnectionDisconnectReason::ReadOnly,
218        }
219    }
220}
221
222/// A request sent through the channel to the background worker.
223///
224/// Most callers send a `Single` frame. Callers that need multiple frames to
225/// land on the wire contiguously (without other tasks' commands interleaving)
226/// -- for example ASKING followed by a migrated command during a cluster
227/// resharding -- send a `Multi` request instead. The worker guarantees that
228/// all frames inside one `Multi` are flushed back-to-back in the same
229/// `execute_pipeline` call.
230enum WorkerRequest {
231    Single {
232        frame: Frame,
233        response_tx: oneshot::Sender<Result<Frame, RedisError>>,
234    },
235    Multi {
236        frames: Vec<Frame>,
237        response_tx: oneshot::Sender<Result<Vec<Frame>, RedisError>>,
238    },
239}
240
241impl WorkerRequest {
242    fn frame_count(&self) -> usize {
243        match self {
244            WorkerRequest::Single { .. } => 1,
245            WorkerRequest::Multi { frames, .. } => frames.len(),
246        }
247    }
248
249    fn fail(self, err: RedisError) {
250        match self {
251            WorkerRequest::Single { response_tx, .. } => {
252                let _ = response_tx.send(Err(err));
253            }
254            WorkerRequest::Multi { response_tx, .. } => {
255                let _ = response_tx.send(Err(err));
256            }
257        }
258    }
259
260    /// Return whether the caller has dropped its response future.
261    ///
262    /// The worker checks this immediately before flattening a batch for the
263    /// wire. Requests cancelled while waiting in `batch_window` must not be
264    /// executed after their caller deadline has already expired.
265    fn response_is_closed(&self) -> bool {
266        match self {
267            WorkerRequest::Single { response_tx, .. } => response_tx.is_closed(),
268            WorkerRequest::Multi { response_tx, .. } => response_tx.is_closed(),
269        }
270    }
271}
272
273/// A `Service<Frame>` that transparently batches concurrent requests into
274/// Redis pipelines for better throughput.
275///
276/// Uses a channel-based approach similar to Tower's `Buffer`:
277///
278/// 1. `call()` sends the request `Frame` plus a oneshot sender through a channel
279/// 2. A background task collects requests for up to `batch_window` duration
280///    or `max_batch_size` requests
281/// 3. The background task sends all collected frames via `execute_pipeline`
282/// 4. Each response is routed back via the corresponding oneshot sender
283///
284/// Compose with [`CommandAdapter`](crate::CommandAdapter) for typed commands:
285///
286/// ```no_run
287/// # use redis_tower::{AutoPipelineConfig, AutoPipelineService, CommandAdapter, RedisConnection};
288/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
289/// # let conn = RedisConnection::connect("127.0.0.1:6379").await?;
290/// # let config = AutoPipelineConfig::default();
291/// let pipeline = AutoPipelineService::new(conn, config);
292/// let shutdown = pipeline.shutdown_handle();
293/// let svc = CommandAdapter::new(pipeline);
294/// # let _ = svc;
295/// # shutdown.shutdown().await;
296/// # Ok(())
297/// # }
298/// ```
299pub struct AutoPipelineService {
300    tx: mpsc::Sender<WorkerRequest>,
301    /// Reservation-based view of the same channel, used for the back-pressure
302    /// path: `poll_ready` reserves a slot via [`PollSender::poll_reserve`] and
303    /// `call` fills it with [`PollSender::send_item`].
304    poll_tx: PollSender<WorkerRequest>,
305    /// When `true`, `call` uses `try_send` and a full queue yields `QueueFull`
306    /// instead of awaiting capacity. Mirrors
307    /// [`AutoPipelineConfig::shed_load_on_full`].
308    shed_load: bool,
309    /// Counts only public service handles. Dropping the final lease wakes and
310    /// cancels a worker that may be inside reconnect backoff or a factory call.
311    lease: WorkerLease,
312    /// Per-clone notification future used to wake a pending `poll_ready` when
313    /// controlled shutdown begins, even while the worker is draining an
314    /// in-flight command and cannot close the request receiver yet.
315    shutdown_notified: Pin<Box<OwnedNotified>>,
316    /// Shared connection-health view. The sender is owned only by the worker,
317    /// so this channel closes when that worker terminates even if service
318    /// clones remain alive.
319    connection_health: watch::Receiver<bool>,
320    worker: Arc<WorkerHandle>,
321}
322
323/// An owned controller for an [`AutoPipelineService`] background worker.
324///
325/// Obtain this handle with [`AutoPipelineService::shutdown_handle`] before the
326/// service is cloned, layered, or stored behind a trait object. Calling
327/// [`shutdown`](Self::shutdown) stops admission across every service clone,
328/// drains requests already accepted by a connected worker, cancels reconnect
329/// or deferred-connect work, and joins the worker. Calls through surviving
330/// service clones then fail with [`RedisError::ConnectionClosed`].
331///
332/// The handle is cloneable and shutdown is idempotent. Concurrent callers all
333/// wait for the same worker. Dropping a handle without calling `shutdown` does
334/// not stop the service.
335#[derive(Clone)]
336#[must_use = "retain the handle and call shutdown() during orderly application shutdown"]
337pub struct AutoPipelineShutdownHandle {
338    control: Arc<WorkerControl>,
339    worker: Arc<WorkerHandle>,
340}
341
342impl std::fmt::Debug for AutoPipelineShutdownHandle {
343    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344        formatter
345            .debug_struct("AutoPipelineShutdownHandle")
346            .field("shutdown_requested", &self.is_shutdown_requested())
347            .field("worker_finished", &self.worker.is_finished())
348            .finish()
349    }
350}
351
352impl AutoPipelineShutdownHandle {
353    /// Stop admission, finish or fail accepted work, and join the worker.
354    ///
355    /// A connected worker drains its accepted queue before exiting. If no
356    /// usable connection exists, queued requests fail with
357    /// [`RedisError::ConnectionClosed`] rather than being replayed during
358    /// shutdown. Repeated and concurrent calls are safe and wait for the same
359    /// terminal worker state.
360    pub async fn shutdown(&self) {
361        self.control.request_shutdown();
362        self.worker.join().await;
363    }
364
365    /// Return whether controlled shutdown has begun.
366    pub fn is_shutdown_requested(&self) -> bool {
367        self.control.is_shutdown_requested()
368    }
369
370    /// Return whether the background worker has finished.
371    pub fn is_finished(&self) -> bool {
372        self.worker.is_finished()
373    }
374}
375
376struct WorkerControl {
377    state: Mutex<WorkerControlState>,
378    shutdown_tx: watch::Sender<bool>,
379    shutdown_notify: Arc<Notify>,
380    event_bus: Option<ConnectionEventBus>,
381}
382
383struct WorkerControlState {
384    handles: usize,
385    worker_running: bool,
386    shutdown_requested: bool,
387    shutdown_published: bool,
388}
389
390impl WorkerControl {
391    fn new(shutdown_tx: watch::Sender<bool>, event_bus: Option<ConnectionEventBus>) -> Self {
392        Self {
393            state: Mutex::new(WorkerControlState {
394                handles: 1,
395                worker_running: true,
396                shutdown_requested: false,
397                shutdown_published: false,
398            }),
399            shutdown_tx,
400            shutdown_notify: Arc::new(Notify::new()),
401            event_bus,
402        }
403    }
404
405    fn add_handle(&self) {
406        let mut state = self
407            .state
408            .lock()
409            .unwrap_or_else(std::sync::PoisonError::into_inner);
410        state.handles = state
411            .handles
412            .checked_add(1)
413            .expect("AutoPipelineService handle count overflowed");
414    }
415
416    fn release_handle(&self) {
417        let should_signal = {
418            let mut state = self
419                .state
420                .lock()
421                .unwrap_or_else(std::sync::PoisonError::into_inner);
422            state.handles = state
423                .handles
424                .checked_sub(1)
425                .expect("AutoPipelineService handle count underflowed");
426            let should_signal = state.handles == 0 && !state.shutdown_requested;
427            if state.handles == 0 {
428                state.shutdown_requested = true;
429                if !state.worker_running {
430                    self.publish_shutdown_locked(&mut state);
431                }
432            }
433            should_signal
434        };
435        if should_signal {
436            self.shutdown_tx.send_replace(true);
437            self.shutdown_notify.notify_waiters();
438        }
439    }
440
441    fn request_shutdown(&self) {
442        let should_signal = {
443            let mut state = self
444                .state
445                .lock()
446                .unwrap_or_else(std::sync::PoisonError::into_inner);
447            let should_signal = !state.shutdown_requested;
448            state.shutdown_requested = true;
449            if !state.worker_running {
450                self.publish_shutdown_locked(&mut state);
451            }
452            should_signal
453        };
454        if should_signal {
455            self.shutdown_tx.send_replace(true);
456            self.shutdown_notify.notify_waiters();
457        }
458    }
459
460    fn shutdown_notified(&self) -> OwnedNotified {
461        Arc::clone(&self.shutdown_notify).notified_owned()
462    }
463
464    fn is_shutdown_requested(&self) -> bool {
465        self.state
466            .lock()
467            .unwrap_or_else(std::sync::PoisonError::into_inner)
468            .shutdown_requested
469    }
470
471    fn handle_count(&self) -> usize {
472        self.state
473            .lock()
474            .unwrap_or_else(std::sync::PoisonError::into_inner)
475            .handles
476    }
477
478    fn worker_finished(&self) {
479        let mut state = self
480            .state
481            .lock()
482            .unwrap_or_else(std::sync::PoisonError::into_inner);
483        state.worker_running = false;
484        if state.shutdown_requested {
485            self.publish_shutdown_locked(&mut state);
486        }
487    }
488
489    fn publish_shutdown_locked(&self, state: &mut WorkerControlState) {
490        if state.shutdown_published {
491            return;
492        }
493        state.shutdown_published = true;
494        if let Some(events) = &self.event_bus {
495            events.publish(ConnectionEvent::Disconnected {
496                reason: ConnectionDisconnectReason::Shutdown,
497            });
498        }
499    }
500
501    /// Run `f` while holding the same lock used by final-handle release.
502    /// This linearizes lifecycle events before shutdown or suppresses them
503    /// after shutdown, so `Shutdown` remains the terminal event.
504    fn with_active_handle<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
505        let state = self
506            .state
507            .lock()
508            .unwrap_or_else(std::sync::PoisonError::into_inner);
509        if state.handles == 0 || state.shutdown_requested {
510            None
511        } else {
512            Some(f())
513        }
514    }
515
516    /// Linearize request admission against explicit or final-handle shutdown.
517    fn with_admission<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
518        let state = self
519            .state
520            .lock()
521            .unwrap_or_else(std::sync::PoisonError::into_inner);
522        if state.shutdown_requested {
523            None
524        } else {
525            Some(f())
526        }
527    }
528}
529
530struct WorkerRunGuard {
531    control: Arc<WorkerControl>,
532}
533
534impl Drop for WorkerRunGuard {
535    fn drop(&mut self) {
536        self.control.worker_finished();
537    }
538}
539
540struct WorkerLifecycle {
541    control: Arc<WorkerControl>,
542    shutdown: watch::Receiver<bool>,
543    connection_health: ConnectionHealthPublisher,
544    _run_guard: WorkerRunGuard,
545}
546
547/// Worker-owned publisher for the current data-connection health.
548///
549/// Dropping the worker always leaves observers with a final `false` snapshot
550/// before closing the watch channel.
551struct ConnectionHealthPublisher {
552    tx: watch::Sender<bool>,
553}
554
555impl ConnectionHealthPublisher {
556    fn new(tx: watch::Sender<bool>) -> Self {
557        Self { tx }
558    }
559
560    fn set(&self, healthy: bool) {
561        self.tx.send_if_modified(|current| {
562            let changed = *current != healthy;
563            *current = healthy;
564            changed
565        });
566    }
567}
568
569impl Drop for ConnectionHealthPublisher {
570    fn drop(&mut self) {
571        self.set(false);
572    }
573}
574
575struct WorkerLease {
576    control: Arc<WorkerControl>,
577}
578
579impl Clone for WorkerLease {
580    fn clone(&self) -> Self {
581        self.control.add_handle();
582        Self {
583            control: Arc::clone(&self.control),
584        }
585    }
586}
587
588impl Drop for WorkerLease {
589    fn drop(&mut self) {
590        self.control.release_handle();
591    }
592}
593
594/// Wrapper around the background task's [`JoinHandle`](tokio::task::JoinHandle)
595/// that emits a warning when dropped without being cleanly shut down.
596///
597/// Stores the join handle behind an async mutex so consuming service shutdown
598/// and independently owned [`AutoPipelineShutdownHandle`] clones can perform
599/// one cancellation-safe, idempotent join.
600struct WorkerHandle {
601    handle: AsyncMutex<Option<tokio::task::JoinHandle<()>>>,
602}
603
604impl WorkerHandle {
605    fn new(handle: tokio::task::JoinHandle<()>) -> Self {
606        Self {
607            handle: AsyncMutex::new(Some(handle)),
608        }
609    }
610
611    /// Returns `true` if the background task has already finished.
612    fn is_finished(&self) -> bool {
613        self.handle
614            .try_lock()
615            .map(|handle| handle.as_ref().is_none_or(|handle| handle.is_finished()))
616            .unwrap_or(false)
617    }
618
619    /// Cancellation-safe, idempotent join shared by every lifecycle handle.
620    async fn join(&self) {
621        let mut handle = self.handle.lock().await;
622        if let Some(handle) = handle.as_mut() {
623            let _ = handle.await;
624        }
625        handle.take();
626    }
627}
628
629impl Drop for WorkerHandle {
630    fn drop(&mut self) {
631        // If the task has not yet finished when the last Arc<WorkerHandle>
632        // is dropped, the JoinHandle is being abandoned. Any in-flight
633        // requests in the pipeline worker may be silently dropped.
634        let unfinished = self
635            .handle
636            .try_lock()
637            .map(|handle| handle.as_ref().is_some_and(|handle| !handle.is_finished()))
638            .unwrap_or(false);
639        if unfinished {
640            warn!(
641                "AutoPipelineService dropped without calling shutdown(); \
642                 background worker may still have queued requests in flight"
643            );
644        }
645    }
646}
647
648impl AutoPipelineService {
649    /// Create a new auto-pipelining service wrapping the given connection.
650    ///
651    /// The connection is moved into a background task that exclusively owns it.
652    /// All requests are sent through a channel and batched automatically.
653    ///
654    /// The service does **not** reconnect if the connection fails -- every
655    /// subsequent request returns [`RedisError::ConnectionClosed`]. Wrap this
656    /// in a reconnect layer, or use [`Self::with_factory`] to build a
657    /// service that rebuilds its own connection on failure.
658    pub fn new(conn: RedisConnection, config: AutoPipelineConfig) -> Self {
659        Self::from_parts(Some(conn), config, ConnSource::Fixed, None, None)
660    }
661
662    /// Create a non-reconnecting auto-pipeline service with lifecycle events.
663    ///
664    /// The bus receives [`ConnectionEvent::Connected`] immediately and one
665    /// [`ConnectionEvent::Disconnected`] transition if the fixed connection
666    /// later fails. This constructor does not add reconnection behavior; use
667    /// [`Self::with_factory_and_events`] for that.
668    pub fn new_with_events(
669        conn: RedisConnection,
670        config: AutoPipelineConfig,
671        events: ConnectionEventBus,
672    ) -> Self {
673        Self::from_parts(Some(conn), config, ConnSource::Fixed, Some(events), None)
674    }
675
676    /// Create a new auto-pipelining service that rebuilds its connection on
677    /// failure using the provided [`ConnectionFactory`].
678    ///
679    /// On pipeline execution failure, in-flight requests receive
680    /// [`RedisError::ConnectionClosed`], then the worker reconnects via the
681    /// factory with exponential backoff governed by `reconnect`. Subsequent
682    /// requests are served by the new connection.
683    ///
684    /// The factory is also the right place to replay session setup
685    /// (AUTH, SELECT, HELLO, READONLY) on every reconnect -- see
686    /// [`UrlConnectionFactory`](crate::reconnect::UrlConnectionFactory)
687    /// for a ready-made AUTH+SELECT factory.
688    pub async fn with_factory(
689        factory: impl ConnectionFactory,
690        config: AutoPipelineConfig,
691        reconnect: AutoPipelineReconnectConfig,
692    ) -> Result<Self, RedisError> {
693        Self::with_factory_inner(factory, config, reconnect, None).await
694    }
695
696    /// Create a factory-backed auto-pipeline service with lifecycle events.
697    ///
698    /// Subscribe before calling this constructor to observe the initial
699    /// connect result. Disconnect and reconnect transitions are published by
700    /// the existing pipeline worker; event delivery does not spawn a separate
701    /// task and never blocks that worker.
702    pub async fn with_factory_and_events(
703        factory: impl ConnectionFactory,
704        config: AutoPipelineConfig,
705        reconnect: AutoPipelineReconnectConfig,
706        events: ConnectionEventBus,
707    ) -> Result<Self, RedisError> {
708        Self::with_factory_inner(factory, config, reconnect, Some(events)).await
709    }
710
711    /// Create an explicitly maintenance-aware factory-backed service.
712    ///
713    /// The initial connection is verified as RESP3 and successfully registers
714    /// `CLIENT MAINT_NOTIFICATIONS` before this constructor returns. Retain the
715    /// returned handle to keep handling enabled.
716    pub async fn with_factory_and_maintenance(
717        factory: impl ConnectionFactory,
718        config: AutoPipelineConfig,
719        reconnect: AutoPipelineReconnectConfig,
720    ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
721        Self::with_factory_and_maintenance_inner(factory, config, reconnect, None).await
722    }
723
724    /// Create an explicitly maintenance-aware service with lifecycle events.
725    pub async fn with_factory_and_maintenance_and_events(
726        factory: impl ConnectionFactory,
727        config: AutoPipelineConfig,
728        reconnect: AutoPipelineReconnectConfig,
729        events: ConnectionEventBus,
730    ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
731        Self::with_factory_and_maintenance_inner(factory, config, reconnect, Some(events)).await
732    }
733
734    async fn with_factory_and_maintenance_inner(
735        factory: impl ConnectionFactory,
736        config: AutoPipelineConfig,
737        reconnect: AutoPipelineReconnectConfig,
738        event_bus: Option<ConnectionEventBus>,
739    ) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
740        let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
741        let (conn, push_rx) =
742            match connect_and_prepare(factory.as_ref(), reconnect.reconnect.connect_timeout, true)
743                .await
744            {
745                Ok((conn, Some(push_rx))) => (conn, push_rx),
746                Ok((_conn, None)) => {
747                    unreachable!("maintenance setup always returns a push receiver")
748                }
749                Err(error) => {
750                    if let Some(events) = &event_bus {
751                        events.publish_with(|| ConnectionEvent::ConnectFailed {
752                            error: Arc::from(error.to_string()),
753                        });
754                    }
755                    return Err(error);
756                }
757            };
758        let listener_id = next_listener_id();
759        let (control_tx, control_rx) = mpsc::unbounded_channel();
760        let source = ConnSource::Factory {
761            factory,
762            reconnect: reconnect.reconnect,
763        };
764        let maintenance = MaintenanceWorker {
765            state: MaintenanceState::new(listener_id),
766            control_rx,
767            push_rx,
768        };
769        let service = Self::from_parts(Some(conn), config, source, event_bus, Some(maintenance));
770        let handle = MaintenanceListenerHandle::new(control_tx, listener_id);
771        Ok((service, handle))
772    }
773
774    async fn with_factory_inner(
775        factory: impl ConnectionFactory,
776        config: AutoPipelineConfig,
777        reconnect: AutoPipelineReconnectConfig,
778        event_bus: Option<ConnectionEventBus>,
779    ) -> Result<Self, RedisError> {
780        let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
781        let conn = match connect_with_timeout(factory.as_ref(), reconnect.reconnect.connect_timeout)
782            .await
783        {
784            Ok(conn) => conn,
785            Err(error) => {
786                if let Some(events) = &event_bus {
787                    events.publish_with(|| ConnectionEvent::ConnectFailed {
788                        error: Arc::from(error.to_string()),
789                    });
790                }
791                return Err(error);
792            }
793        };
794        let source = ConnSource::Factory {
795            factory,
796            reconnect: reconnect.reconnect,
797        };
798        Ok(Self::from_parts(
799            Some(conn),
800            config,
801            source,
802            event_bus,
803            None,
804        ))
805    }
806
807    /// Create a factory-backed service without opening a connection yet.
808    ///
809    /// Construction is synchronous and performs no network I/O. The first
810    /// accepted request invokes the factory and waits for that connection
811    /// attempt before it is sent. If the attempt fails, that request receives
812    /// [`RedisError::ConnectionClosed`]; the worker remains alive and the next
813    /// request makes a fresh attempt. Once connected, failures use `reconnect`
814    /// in exactly the same way as [`Self::with_factory`].
815    ///
816    /// Connection health starts as `false` and becomes `true` immediately
817    /// before the first [`ConnectionEvent::Connected`] event is published.
818    /// With an event bus, a failed deferred attempt publishes
819    /// [`ConnectionEvent::ConnectFailed`]. No `Connected` event is emitted
820    /// merely by constructing the service.
821    ///
822    /// # Panics
823    ///
824    /// Panics when called outside an entered Tokio runtime because the
825    /// lightweight request worker is spawned during construction.
826    pub fn with_lazy_factory(
827        factory: impl ConnectionFactory,
828        config: AutoPipelineConfig,
829        reconnect: AutoPipelineReconnectConfig,
830    ) -> Self {
831        Self::with_lazy_factory_inner(factory, config, reconnect, None)
832    }
833
834    /// Create a lazily connected factory-backed service with lifecycle events.
835    ///
836    /// Subscribe to `events` before this call to observe the first deferred
837    /// connection result. Construction itself publishes no connection event.
838    ///
839    /// # Panics
840    ///
841    /// Panics when called outside an entered Tokio runtime.
842    pub fn with_lazy_factory_and_events(
843        factory: impl ConnectionFactory,
844        config: AutoPipelineConfig,
845        reconnect: AutoPipelineReconnectConfig,
846        events: ConnectionEventBus,
847    ) -> Self {
848        Self::with_lazy_factory_inner(factory, config, reconnect, Some(events))
849    }
850
851    fn with_lazy_factory_inner(
852        factory: impl ConnectionFactory,
853        config: AutoPipelineConfig,
854        reconnect: AutoPipelineReconnectConfig,
855        event_bus: Option<ConnectionEventBus>,
856    ) -> Self {
857        let source = ConnSource::Factory {
858            factory: Arc::new(factory),
859            reconnect: reconnect.reconnect,
860        };
861        Self::from_parts(None, config, source, event_bus, None)
862    }
863
864    fn from_parts(
865        conn: Option<RedisConnection>,
866        config: AutoPipelineConfig,
867        source: ConnSource,
868        event_bus: Option<ConnectionEventBus>,
869        maintenance: Option<MaintenanceWorker>,
870    ) -> Self {
871        let (tx, rx) = mpsc::channel(config.queue_capacity);
872        let poll_tx = PollSender::new(tx.clone());
873        let shed_load = config.shed_load_on_full;
874        let (shutdown_tx, shutdown_rx) = watch::channel(false);
875        let initially_connected = conn.is_some();
876        let (connection_health_tx, connection_health) = watch::channel(initially_connected);
877        let control = Arc::new(WorkerControl::new(shutdown_tx, event_bus.clone()));
878        let lease = WorkerLease {
879            control: Arc::clone(&control),
880        };
881        if initially_connected && let Some(events) = &event_bus {
882            events.publish(ConnectionEvent::Connected);
883        }
884        let lifecycle = WorkerLifecycle {
885            control: Arc::clone(&control),
886            shutdown: shutdown_rx,
887            connection_health: ConnectionHealthPublisher::new(connection_health_tx),
888            _run_guard: WorkerRunGuard {
889                control: Arc::clone(&control),
890            },
891        };
892        let runtime = tokio::runtime::Handle::try_current()
893            .expect("AutoPipelineService must be constructed inside an entered Tokio runtime");
894        let handle = runtime.spawn(pipeline_worker(
895            rx,
896            conn,
897            config,
898            source,
899            event_bus,
900            lifecycle,
901            maintenance,
902        ));
903        Self {
904            tx,
905            poll_tx,
906            shed_load,
907            lease,
908            shutdown_notified: Box::pin(control.shutdown_notified()),
909            connection_health,
910            worker: Arc::new(WorkerHandle::new(handle)),
911        }
912    }
913
914    /// Send multiple frames through the service as a single atomic batch.
915    ///
916    /// The worker guarantees that all frames in the supplied slice are
917    /// flushed back-to-back in one [`RedisConnection::execute_pipeline`]
918    /// call, with no interleaving from other concurrent callers. This is
919    /// what you want for sequences like `ASKING` + the migrated command
920    /// during cluster resharding, where ordering on a single connection
921    /// matters.
922    ///
923    /// Returns one response frame per input frame, in order. If any frame's
924    /// response is an error, the overall call still returns the full
925    /// response vector -- error inspection is left to the caller.
926    pub async fn call_pipeline(&mut self, frames: Vec<Frame>) -> Result<Vec<Frame>, RedisError> {
927        if frames.is_empty() {
928            return Ok(Vec::new());
929        }
930        let (resp_tx, resp_rx) = oneshot::channel();
931        let request = WorkerRequest::Multi {
932            frames,
933            response_tx: resp_tx,
934        };
935        if self.shed_load {
936            self.lease
937                .control
938                .with_admission(|| {
939                    self.tx.try_send(request).map_err(|e| match e {
940                        tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
941                        tokio::sync::mpsc::error::TrySendError::Closed(_) => {
942                            RedisError::ConnectionClosed
943                        }
944                    })
945                })
946                .ok_or(RedisError::ConnectionClosed)??;
947        } else {
948            // Back-pressure: await a free slot rather than failing fast.
949            // Create the notification future before checking state so a
950            // shutdown racing this admission cannot be missed.
951            let shutdown = self.lease.control.shutdown_notified();
952            if self.lease.control.is_shutdown_requested() {
953                return Err(RedisError::ConnectionClosed);
954            }
955            let permit = tokio::select! {
956                biased;
957                () = shutdown => return Err(RedisError::ConnectionClosed),
958                permit = self.tx.reserve() => {
959                    permit.map_err(|_| RedisError::ConnectionClosed)?
960                }
961            };
962            self.lease
963                .control
964                .with_admission(|| permit.send(request))
965                .ok_or(RedisError::ConnectionClosed)?;
966        }
967        resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
968    }
969
970    /// Send multiple frames through the queue slot reserved by
971    /// [`Service::poll_ready`] as one atomic worker request.
972    ///
973    /// This is the multi-frame counterpart to [`Service::call`]. It exists for
974    /// frame-level middleware that must prepend connection-local setup to one
975    /// command without allowing another caller to interleave between them. In
976    /// particular, client-side caching in opt-in mode uses it for `CLIENT
977    /// CACHING YES` followed by the cacheable read.
978    ///
979    /// In back-pressure mode the caller must first drive `poll_ready` to
980    /// `Ready(Ok(()))`; this method consumes that exact reservation. In
981    /// load-shedding mode `poll_ready` does not reserve capacity and this
982    /// method follows [`Service::call`] by using `try_send`.
983    pub(crate) fn call_reserved_pipeline(
984        &mut self,
985        frames: Vec<Frame>,
986    ) -> Pin<Box<dyn Future<Output = Result<Vec<Frame>, RedisError>> + Send>> {
987        if frames.is_empty() {
988            // Release a back-pressure reservation even though there is no
989            // worker request to fill it with.
990            self.release_reservation();
991            return Box::pin(async { Ok(Vec::new()) });
992        }
993
994        let (resp_tx, resp_rx) = oneshot::channel();
995        let request = WorkerRequest::Multi {
996            frames,
997            response_tx: resp_tx,
998        };
999
1000        let send_result = if self.shed_load {
1001            self.lease
1002                .control
1003                .with_admission(|| {
1004                    self.tx.try_send(request).map_err(|e| match e {
1005                        tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
1006                        tokio::sync::mpsc::error::TrySendError::Closed(_) => {
1007                            RedisError::ConnectionClosed
1008                        }
1009                    })
1010                })
1011                .unwrap_or(Err(RedisError::ConnectionClosed))
1012        } else {
1013            self.lease
1014                .control
1015                .with_admission(|| {
1016                    self.poll_tx
1017                        .send_item(request)
1018                        .map_err(|_| RedisError::ConnectionClosed)
1019                })
1020                .unwrap_or_else(|| {
1021                    let _ = self.poll_tx.abort_send();
1022                    Err(RedisError::ConnectionClosed)
1023                })
1024        };
1025
1026        Box::pin(async move {
1027            send_result?;
1028            resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
1029        })
1030    }
1031
1032    /// Release a queue slot previously reserved by [`Service::poll_ready`]
1033    /// without submitting a worker request.
1034    ///
1035    /// Frame middleware uses this when it satisfies a request locally after
1036    /// readiness was already established. Client-side cache hits are the
1037    /// canonical case: they must return the cached response while restoring
1038    /// the auto-pipeline queue's full capacity.
1039    pub(crate) fn release_reservation(&mut self) {
1040        let _ = self.poll_tx.abort_send();
1041    }
1042
1043    /// Return an owned controller for this service's shared worker.
1044    ///
1045    /// Retain the handle before moving the service into Tower layers, a router,
1046    /// or a trait object. Unlike [`Self::shutdown`], the handle stops the worker
1047    /// even while public service clones still exist. Shutdown is idempotent and
1048    /// all handle clones wait for the same queue drain and worker join.
1049    pub fn shutdown_handle(&self) -> AutoPipelineShutdownHandle {
1050        AutoPipelineShutdownHandle {
1051            control: Arc::clone(&self.lease.control),
1052            worker: Arc::clone(&self.worker),
1053        }
1054    }
1055
1056    /// Gracefully shut down the pipeline service.
1057    ///
1058    /// Drops this instance's sender half. If this is the last live service
1059    /// clone, it also signals the background worker to close its receiver,
1060    /// reject any late sends, drain requests that were already accepted, and
1061    /// exit cleanly. The method waits for that final worker exit.
1062    ///
1063    /// If other clones are still alive, returns immediately -- the worker
1064    /// continues running until the last clone shuts down or is dropped.
1065    /// When lifecycle events are configured, the service publishes exactly one
1066    /// [`ConnectionEvent::Disconnected`] with
1067    /// [`ConnectionDisconnectReason::Shutdown`] after the worker observes that
1068    /// the final public handle closed. An earlier unexpected `Disconnected`
1069    /// event does not suppress this distinct terminal transition. `shutdown()`
1070    /// waits for normal queued work to drain and for this event to be emitted.
1071    /// If the worker is reconnecting, the final handle cancels its backoff or
1072    /// in-progress factory call before the worker exits.
1073    ///
1074    /// For clean application shutdown, prefer calling `shutdown()` over
1075    /// simply dropping the service. When the final concrete clone cannot be
1076    /// recovered, retain [`Self::shutdown_handle`] before distributing clones.
1077    pub async fn shutdown(self) {
1078        let control = Arc::clone(&self.lease.control);
1079        let worker = Arc::clone(&self.worker);
1080        // Drop both sender handles first: decrements the sender count. When all
1081        // senders are gone the worker's `recv()` returns `None` and the worker
1082        // exits after flushing any remaining batch. `poll_tx` holds its own
1083        // clone of the sender, so it must be dropped too or the worker await
1084        // below would hang.
1085        drop(self.tx);
1086        drop(self.poll_tx);
1087        // The final public service lease signals the worker independently of
1088        // the request channel, so reconnect backoff and hanging factories are
1089        // cancellation-safe too.
1090        drop(self.lease);
1091        drop(self.worker);
1092        // The final public service handle joins the worker even when a
1093        // shutdown handle is retained elsewhere. Non-final clones preserve
1094        // the historical behavior and return without stopping shared work.
1095        if control.handle_count() == 0 {
1096            worker.join().await;
1097        }
1098    }
1099
1100    /// Returns `true` while the background worker task is still running.
1101    ///
1102    /// A factory-backed worker exits only after exhausting its reconnect budget
1103    /// (or on a clean [`shutdown`](Self::shutdown)); once it has, this service
1104    /// can no longer serve requests. A cluster client uses this to detect a
1105    /// per-node worker that gave up on a dead address so it can replace the
1106    /// service during a topology refresh.
1107    pub fn is_alive(&self) -> bool {
1108        !self.worker.is_finished()
1109    }
1110
1111    /// Return the worker's current data-connection health snapshot.
1112    ///
1113    /// This is public only so sibling workspace clients can fail closed when
1114    /// connection-local server state is lost. In particular, a cluster cache
1115    /// supervisor must not serve hits while any tracked node is disconnected.
1116    #[doc(hidden)]
1117    pub fn is_connection_healthy(&self) -> bool {
1118        *self.connection_health.borrow()
1119    }
1120
1121    /// Subscribe to data-connection health transitions.
1122    ///
1123    /// The initial value is `true` for a service given an established
1124    /// connection and `false` for a lazily connected service. Failures publish
1125    /// `false` before request responders or disconnect events are notified;
1126    /// initial connection and factory-backed reconnection publish `true` only
1127    /// after a connection is installed. The channel closes when the worker
1128    /// terminates. This is public only for sibling workspace clients that own
1129    /// a coordinated lifecycle across multiple data connections.
1130    #[doc(hidden)]
1131    pub fn subscribe_connection_health(&self) -> watch::Receiver<bool> {
1132        self.connection_health.clone()
1133    }
1134}
1135
1136/// Background task that collects requests and executes them as pipelines.
1137///
1138/// Batch size is measured in *frames*, not requests, so a single `Multi`
1139/// request carrying N frames counts as N toward `max_batch_size`. This keeps
1140/// the effective flush size stable regardless of how many frames individual
1141/// callers send.
1142async fn pipeline_worker(
1143    mut rx: mpsc::Receiver<WorkerRequest>,
1144    mut conn: Option<RedisConnection>,
1145    config: AutoPipelineConfig,
1146    source: ConnSource,
1147    event_bus: Option<ConnectionEventBus>,
1148    lifecycle: WorkerLifecycle,
1149    maintenance: Option<MaintenanceWorker>,
1150) {
1151    let WorkerLifecycle {
1152        control,
1153        mut shutdown,
1154        connection_health,
1155        _run_guard,
1156    } = lifecycle;
1157    let mut outage_reported = false;
1158    let mut shutting_down = false;
1159    let (mut maintenance_state, mut maintenance_control_rx, mut maintenance_push_rx) =
1160        match maintenance {
1161            Some(maintenance) => (
1162                Some(maintenance.state),
1163                Some(maintenance.control_rx),
1164                Some(maintenance.push_rx),
1165            ),
1166            None => (None, None, None),
1167        };
1168    'worker: loop {
1169        drain_maintenance_control(
1170            &mut maintenance_state,
1171            &mut maintenance_control_rx,
1172            &mut maintenance_push_rx,
1173        );
1174        if maintenance_state
1175            .as_ref()
1176            .and_then(|state| state.pending.as_ref())
1177            .is_some()
1178        {
1179            match wait_for_pending_handoff(
1180                &mut conn,
1181                &mut maintenance_state,
1182                &mut maintenance_control_rx,
1183                &mut maintenance_push_rx,
1184                event_bus.as_ref(),
1185                &mut shutdown,
1186            )
1187            .await
1188            {
1189                PendingHandoffOutcome::Cancelled => continue,
1190                PendingHandoffOutcome::Shutdown => {
1191                    shutting_down = true;
1192                    rx.close();
1193                    disable_maintenance(
1194                        &mut maintenance_state,
1195                        &mut maintenance_control_rx,
1196                        &mut maintenance_push_rx,
1197                    );
1198                    continue;
1199                }
1200                PendingHandoffOutcome::ConnectionFailed(error) => {
1201                    clear_pending_handoff(&mut maintenance_state);
1202                    connection_health.set(false);
1203                    let failure = PipelineFailure::Connection(error);
1204                    publish_worker_outage(
1205                        event_bus.as_ref(),
1206                        &mut outage_reported,
1207                        failure.event_reason(),
1208                        &control,
1209                    );
1210                    let ConnSource::Factory { factory, reconnect } = &source else {
1211                        fail_queued_requests(&mut rx);
1212                        return;
1213                    };
1214                    let maintenance_enabled = maintenance_state
1215                        .as_ref()
1216                        .is_some_and(|state| state.enabled);
1217                    if !reconnect_worker_connection(
1218                        conn.as_mut()
1219                            .expect("maintenance listener requires a connection"),
1220                        &mut rx,
1221                        factory.as_ref(),
1222                        reconnect,
1223                        event_bus.as_ref(),
1224                        &mut shutdown,
1225                        &control,
1226                        &connection_health,
1227                        &mut outage_reported,
1228                        &mut maintenance_push_rx,
1229                        maintenance_enabled,
1230                    )
1231                    .await
1232                    {
1233                        return;
1234                    }
1235                    continue;
1236                }
1237                PendingHandoffOutcome::Ready { sequence } => {
1238                    if let Some(state) = &mut maintenance_state {
1239                        state.pending = None;
1240                    }
1241                    connection_health.set(false);
1242                    publish_worker_outage(
1243                        event_bus.as_ref(),
1244                        &mut outage_reported,
1245                        ConnectionDisconnectReason::MaintenanceHandoff { sequence },
1246                        &control,
1247                    );
1248                    let ConnSource::Factory { factory, reconnect } = &source else {
1249                        fail_queued_requests(&mut rx);
1250                        return;
1251                    };
1252                    let maintenance_enabled = maintenance_state
1253                        .as_ref()
1254                        .is_some_and(|state| state.enabled);
1255                    if !reconnect_worker_connection(
1256                        conn.as_mut()
1257                            .expect("maintenance listener requires a connection"),
1258                        &mut rx,
1259                        factory.as_ref(),
1260                        reconnect,
1261                        event_bus.as_ref(),
1262                        &mut shutdown,
1263                        &control,
1264                        &connection_health,
1265                        &mut outage_reported,
1266                        &mut maintenance_push_rx,
1267                        maintenance_enabled,
1268                    )
1269                    .await
1270                    {
1271                        return;
1272                    }
1273                    continue;
1274                }
1275            }
1276        }
1277
1278        // Wait for the first request or final-handle shutdown. Closing the
1279        // receiver rejects any sender retained outside a public service
1280        // handle while still allowing already-buffered requests to drain.
1281        let first = if shutting_down {
1282            // Admission is already disabled under WorkerControl's state lock,
1283            // so every accepted request is synchronously visible in the
1284            // receiver buffer. Do not await unused permits held by surviving
1285            // clones; they were never accepted and cannot be filled now.
1286            rx.try_recv().ok()
1287        } else {
1288            loop {
1289                tokio::select! {
1290                    biased;
1291                    () = wait_for_shutdown(&mut shutdown) => {
1292                        shutting_down = true;
1293                        rx.close();
1294                        break rx.try_recv().ok();
1295                    }
1296                    control_message = recv_maintenance_control(&mut maintenance_control_rx) => {
1297                        apply_maintenance_control(
1298                            control_message,
1299                            &mut maintenance_state,
1300                            &mut maintenance_control_rx,
1301                            &mut maintenance_push_rx,
1302                        );
1303                    }
1304                    push = recv_maintenance_push(&mut maintenance_push_rx) => {
1305                        if handle_maintenance_push_result(
1306                            push,
1307                            &mut maintenance_state,
1308                            &mut maintenance_push_rx,
1309                            event_bus.as_ref(),
1310                        ).is_err() {
1311                            connection_health.set(false);
1312                            let failure = PipelineFailure::Connection(
1313                                maintenance_feed_error(),
1314                            );
1315                            publish_worker_outage(
1316                                event_bus.as_ref(),
1317                                &mut outage_reported,
1318                                failure.event_reason(),
1319                                &control,
1320                            );
1321                            let ConnSource::Factory { factory, reconnect } = &source else {
1322                                fail_queued_requests(&mut rx);
1323                                return;
1324                            };
1325                            let maintenance_enabled = maintenance_state
1326                                .as_ref()
1327                                .is_some_and(|state| state.enabled);
1328                            if !reconnect_worker_connection(
1329                                conn.as_mut().expect("maintenance feed requires a connection"),
1330                                &mut rx,
1331                                factory.as_ref(),
1332                                reconnect,
1333                                event_bus.as_ref(),
1334                                &mut shutdown,
1335                                &control,
1336                                &connection_health,
1337                                &mut outage_reported,
1338                                &mut maintenance_push_rx,
1339                                maintenance_enabled,
1340                            ).await {
1341                                return;
1342                            }
1343                        }
1344                        if maintenance_state
1345                            .as_ref()
1346                            .and_then(|state| state.pending.as_ref())
1347                            .is_some()
1348                        {
1349                            continue 'worker;
1350                        }
1351                    }
1352                    request = rx.recv() => break request,
1353                    idle_result = read_idle_push_if_connected(&mut conn) => match idle_result {
1354                        Ok(()) => {
1355                            // A push was routed to its subscriber. Re-enter the
1356                            // biased select so queued requests retain priority
1357                            // over further unsolicited traffic.
1358                        }
1359                        Err(error) => {
1360                            clear_pending_handoff(&mut maintenance_state);
1361                            connection_health.set(false);
1362                            let failure = PipelineFailure::Connection(error);
1363                            publish_worker_outage(
1364                                event_bus.as_ref(),
1365                                &mut outage_reported,
1366                                failure.event_reason(),
1367                                &control,
1368                            );
1369                            match &source {
1370                                ConnSource::Fixed => {
1371                                    // An idle failure proves this fixed
1372                                    // transport cannot serve future work.
1373                                    // Close the queue and reject anything that
1374                                    // raced with the socket notification.
1375                                    rx.close();
1376                                    while let Ok(request) = rx.try_recv() {
1377                                        request.fail(RedisError::ConnectionClosed);
1378                                    }
1379                                    return;
1380                                }
1381                                ConnSource::Factory { factory, reconnect } => {
1382                                    let maintenance_enabled = maintenance_state
1383                                        .as_ref()
1384                                        .is_some_and(|state| state.enabled);
1385                                    if !reconnect_worker_connection(
1386                                        conn.as_mut().expect("idle failure requires a connection"),
1387                                        &mut rx,
1388                                        factory.as_ref(),
1389                                        reconnect,
1390                                        event_bus.as_ref(),
1391                                        &mut shutdown,
1392                                        &control,
1393                                        &connection_health,
1394                                        &mut outage_reported,
1395                                        &mut maintenance_push_rx,
1396                                        maintenance_enabled,
1397                                    )
1398                                    .await
1399                                    {
1400                                        return;
1401                                    }
1402                                }
1403                            }
1404                        }
1405                    },
1406                }
1407            }
1408        };
1409        let first = match first {
1410            Some(req) => req,
1411            None => break, // channel closed, all senders dropped
1412        };
1413
1414        let mut frame_count = first.frame_count();
1415        let mut batch: Vec<WorkerRequest> = vec![first];
1416
1417        // Drain any immediately-available requests without waiting.
1418        // This handles the high-concurrency case where multiple requests
1419        // arrive between flushes.
1420        while frame_count < config.max_batch_size {
1421            match rx.try_recv() {
1422                Ok(req) => {
1423                    frame_count += req.frame_count();
1424                    batch.push(req);
1425                }
1426                Err(_) => break,
1427            }
1428        }
1429
1430        // If we haven't filled the batch and the window is non-zero,
1431        // wait briefly for more requests to arrive.
1432        let mut channel_closed = false;
1433        if frame_count < config.max_batch_size && !config.batch_window.is_zero() {
1434            let deadline = tokio::time::Instant::now() + config.batch_window;
1435            loop {
1436                if frame_count >= config.max_batch_size {
1437                    break;
1438                }
1439
1440                if shutting_down {
1441                    match rx.try_recv() {
1442                        Ok(req) => {
1443                            frame_count += req.frame_count();
1444                            batch.push(req);
1445                            continue;
1446                        }
1447                        Err(_) => break,
1448                    }
1449                }
1450
1451                tokio::select! {
1452                    biased;
1453                    () = wait_for_shutdown(&mut shutdown) => {
1454                        shutting_down = true;
1455                        rx.close();
1456                    }
1457                    request = rx.recv() => match request {
1458                        Some(req) => {
1459                            frame_count += req.frame_count();
1460                            batch.push(req);
1461                        }
1462                        None => {
1463                            channel_closed = true;
1464                            break;
1465                        }
1466                    },
1467                    () = tokio::time::sleep_until(deadline) => break,
1468                }
1469            }
1470        }
1471
1472        if conn.is_none() {
1473            let ConnSource::Factory { factory, reconnect } = &source else {
1474                unreachable!("a fixed worker always starts with a connection");
1475            };
1476            let connect = connect_with_timeout(factory.as_ref(), reconnect.connect_timeout);
1477            tokio::pin!(connect);
1478            let connect_result = tokio::select! {
1479                biased;
1480                () = wait_for_shutdown(&mut shutdown) => {
1481                    shutting_down = true;
1482                    rx.close();
1483                    fail_requests(batch);
1484                    continue;
1485                }
1486                result = &mut connect => result,
1487            };
1488            match connect_result {
1489                Ok(new_conn) => {
1490                    let connected = control.with_active_handle(|| {
1491                        connection_health.set(true);
1492                        if let Some(events) = &event_bus {
1493                            events.publish(ConnectionEvent::Connected);
1494                        }
1495                    });
1496                    if connected.is_none() {
1497                        fail_requests(batch);
1498                        return;
1499                    }
1500                    conn = Some(new_conn);
1501                }
1502                Err(error) => {
1503                    control.with_active_handle(|| {
1504                        if let Some(events) = &event_bus {
1505                            events.publish_with(|| ConnectionEvent::ConnectFailed {
1506                                error: Arc::from(error.to_string()),
1507                            });
1508                        }
1509                    });
1510                    fail_requests(batch);
1511                    continue;
1512                }
1513            }
1514        }
1515
1516        let flush_result = flush_batch(
1517            conn.as_mut().expect("connection established before flush"),
1518            batch,
1519            config.response_timeout,
1520            config.metrics_recorder.as_ref(),
1521            config.reconnect_on_readonly,
1522            &connection_health,
1523        )
1524        .await;
1525
1526        if channel_closed {
1527            if let Err(failure) = flush_result {
1528                publish_worker_outage(
1529                    event_bus.as_ref(),
1530                    &mut outage_reported,
1531                    failure.event_reason(),
1532                    &control,
1533                );
1534            }
1535            return;
1536        }
1537
1538        if let Err(failure) = flush_result {
1539            // A listener disable requested while this batch was in flight
1540            // wins over pushes decoded by the failed exchange. That prevents
1541            // an obsolete MOVING from extending shutdown through a
1542            // maintenance-aware reconnect campaign.
1543            drain_maintenance_control(
1544                &mut maintenance_state,
1545                &mut maintenance_control_rx,
1546                &mut maintenance_push_rx,
1547            );
1548            drain_available_maintenance_pushes(
1549                &mut maintenance_state,
1550                &mut maintenance_push_rx,
1551                event_bus.as_ref(),
1552            );
1553            clear_pending_handoff(&mut maintenance_state);
1554            publish_worker_outage(
1555                event_bus.as_ref(),
1556                &mut outage_reported,
1557                failure.event_reason(),
1558                &control,
1559            );
1560            // Pipeline execution failed. Either give up (Fixed source) or
1561            // reconnect via factory and keep serving.
1562            match &source {
1563                ConnSource::Fixed => {
1564                    // Current behavior: leave the worker running on the dead
1565                    // connection so any future batches also fail-fast and
1566                    // upstream retry layers can notice.
1567                }
1568                ConnSource::Factory { factory, reconnect } => {
1569                    let maintenance_enabled = maintenance_state
1570                        .as_ref()
1571                        .is_some_and(|state| state.enabled);
1572                    if !reconnect_worker_connection(
1573                        conn.as_mut().expect("failed flush requires a connection"),
1574                        &mut rx,
1575                        factory.as_ref(),
1576                        reconnect,
1577                        event_bus.as_ref(),
1578                        &mut shutdown,
1579                        &control,
1580                        &connection_health,
1581                        &mut outage_reported,
1582                        &mut maintenance_push_rx,
1583                        maintenance_enabled,
1584                    )
1585                    .await
1586                    {
1587                        return;
1588                    }
1589                }
1590            }
1591        }
1592    }
1593}
1594
1595/// Wait for an unsolicited push only while a connection exists.
1596///
1597/// A lazy worker has no socket to poll before its first command, so this stays
1598/// pending and lets the request/shutdown branches of the surrounding select
1599/// drive progress.
1600async fn read_idle_push_if_connected(conn: &mut Option<RedisConnection>) -> Result<(), RedisError> {
1601    match conn {
1602        Some(conn) => conn.read_idle_push().await,
1603        None => futures::future::pending().await,
1604    }
1605}
1606
1607enum PendingHandoffOutcome {
1608    Cancelled,
1609    Ready { sequence: u64 },
1610    ConnectionFailed(RedisError),
1611    Shutdown,
1612}
1613
1614async fn recv_maintenance_control(
1615    receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1616) -> Option<MaintenanceControl> {
1617    match receiver {
1618        Some(receiver) => receiver.recv().await,
1619        None => futures::future::pending().await,
1620    }
1621}
1622
1623async fn recv_maintenance_push(
1624    receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1625) -> Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError> {
1626    match receiver {
1627        Some(receiver) => receiver.recv().await,
1628        None => futures::future::pending().await,
1629    }
1630}
1631
1632fn drain_maintenance_control(
1633    state: &mut Option<MaintenanceState>,
1634    receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1635    pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1636) {
1637    loop {
1638        let Some(control) = receiver.as_mut() else {
1639            return;
1640        };
1641        match control.try_recv() {
1642            Ok(message) => apply_maintenance_control(Some(message), state, receiver, pushes),
1643            Err(mpsc::error::TryRecvError::Empty) => return,
1644            Err(mpsc::error::TryRecvError::Disconnected) => {
1645                *receiver = None;
1646                if let Some(state) = state {
1647                    state.enabled = false;
1648                    state.pending = None;
1649                }
1650                *pushes = None;
1651                return;
1652            }
1653        }
1654    }
1655}
1656
1657fn clear_pending_handoff(state: &mut Option<MaintenanceState>) {
1658    if let Some(state) = state {
1659        state.pending = None;
1660    }
1661}
1662
1663fn disable_maintenance(
1664    state: &mut Option<MaintenanceState>,
1665    receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1666    pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1667) {
1668    if let Some(state) = state {
1669        state.enabled = false;
1670        state.pending = None;
1671    }
1672    *receiver = None;
1673    *pushes = None;
1674}
1675
1676fn apply_maintenance_control(
1677    message: Option<MaintenanceControl>,
1678    state: &mut Option<MaintenanceState>,
1679    receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1680    pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1681) {
1682    let Some(message) = message else {
1683        *receiver = None;
1684        return;
1685    };
1686    match message {
1687        MaintenanceControl::Disable { listener_id, ack } => {
1688            if state
1689                .as_ref()
1690                .is_some_and(|state| state.listener_id == listener_id)
1691            {
1692                disable_maintenance(state, receiver, pushes);
1693            }
1694            if let Some(ack) = ack {
1695                let _ = ack.send(());
1696            }
1697        }
1698    }
1699}
1700
1701fn publish_maintenance_notification(
1702    event_bus: Option<&ConnectionEventBus>,
1703    notification: ParsedMaintenanceNotification,
1704) {
1705    if let Some(events) = event_bus {
1706        events.publish(ConnectionEvent::MaintenanceNotification {
1707            kind: notification.kind(),
1708            sequence: notification.sequence(),
1709            ttl: notification.ttl(),
1710        });
1711    }
1712}
1713
1714fn handle_maintenance_push_result(
1715    push: Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError>,
1716    state: &mut Option<MaintenanceState>,
1717    receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1718    event_bus: Option<&ConnectionEventBus>,
1719) -> Result<(), ()> {
1720    match push {
1721        Ok(push) => {
1722            if let Some(state) = state
1723                && let Some(received) = parse_maintenance_push(push)
1724                && let Some(notification) = state.accept(received)
1725            {
1726                publish_maintenance_notification(event_bus, notification);
1727            }
1728            Ok(())
1729        }
1730        Err(_) => {
1731            *receiver = None;
1732            Err(())
1733        }
1734    }
1735}
1736
1737fn drain_available_maintenance_pushes(
1738    state: &mut Option<MaintenanceState>,
1739    receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1740    event_bus: Option<&ConnectionEventBus>,
1741) {
1742    loop {
1743        let Some(pushes) = receiver.as_mut() else {
1744            return;
1745        };
1746        match pushes.try_recv() {
1747            Ok(push) => {
1748                let _ = handle_maintenance_push_result(Ok(push), state, receiver, event_bus);
1749            }
1750            Err(tokio::sync::broadcast::error::TryRecvError::Empty) => return,
1751            Err(tokio::sync::broadcast::error::TryRecvError::Lagged(_)) => {
1752                // The connection is already being discarded. Advance to and
1753                // record every notification still retained by the bounded
1754                // feed so a replay on the replacement is deduplicated.
1755            }
1756            Err(tokio::sync::broadcast::error::TryRecvError::Closed) => {
1757                *receiver = None;
1758                return;
1759            }
1760        }
1761    }
1762}
1763
1764fn maintenance_feed_error() -> RedisError {
1765    RedisError::UnexpectedResponse {
1766        expected: "a complete maintenance-notification push feed",
1767        actual: "maintenance push feed closed or overflowed".to_owned(),
1768    }
1769}
1770
1771async fn connect_and_prepare(
1772    factory: &dyn ConnectionFactory,
1773    timeout: Option<Duration>,
1774    maintenance_enabled: bool,
1775) -> Result<
1776    (
1777        RedisConnection,
1778        Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1779    ),
1780    RedisError,
1781> {
1782    let prepare = async {
1783        let mut connection = factory.connect().await?;
1784        let pushes = if maintenance_enabled {
1785            Some(register_connection(&mut connection).await?)
1786        } else {
1787            None
1788        };
1789        Ok((connection, pushes))
1790    };
1791    match timeout {
1792        Some(timeout) => tokio::time::timeout(timeout, prepare)
1793            .await
1794            .map_err(|_| RedisError::ConnectTimeout)?,
1795        None => prepare.await,
1796    }
1797}
1798
1799async fn wait_for_pending_handoff(
1800    conn: &mut Option<RedisConnection>,
1801    state: &mut Option<MaintenanceState>,
1802    control_rx: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
1803    push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1804    event_bus: Option<&ConnectionEventBus>,
1805    shutdown: &mut watch::Receiver<bool>,
1806) -> PendingHandoffOutcome {
1807    loop {
1808        let Some(pending) = state.as_ref().and_then(|state| state.pending.as_ref()) else {
1809            return PendingHandoffOutcome::Cancelled;
1810        };
1811        let deadline = pending.deadline;
1812        let sequence = pending.sequence;
1813        if deadline <= tokio::time::Instant::now() {
1814            return PendingHandoffOutcome::Ready { sequence };
1815        }
1816
1817        tokio::select! {
1818            biased;
1819            () = wait_for_shutdown(shutdown) => return PendingHandoffOutcome::Shutdown,
1820            control = recv_maintenance_control(control_rx) => {
1821                apply_maintenance_control(control, state, control_rx, push_rx);
1822                if state.as_ref().and_then(|state| state.pending.as_ref()).is_none() {
1823                    return PendingHandoffOutcome::Cancelled;
1824                }
1825            }
1826            push = recv_maintenance_push(push_rx) => {
1827                if handle_maintenance_push_result(push, state, push_rx, event_bus).is_err() {
1828                    return PendingHandoffOutcome::ConnectionFailed(maintenance_feed_error());
1829                }
1830            }
1831            result = read_idle_push_if_connected(conn) => {
1832                if let Err(error) = result {
1833                    return PendingHandoffOutcome::ConnectionFailed(error);
1834                }
1835            }
1836            () = tokio::time::sleep_until(deadline) => {
1837                // Re-read the state on the next loop because another MOVING
1838                // notification may have installed an earlier deadline.
1839            }
1840        }
1841    }
1842}
1843
1844fn fail_requests(requests: Vec<WorkerRequest>) {
1845    for request in requests {
1846        request.fail(RedisError::ConnectionClosed);
1847    }
1848}
1849
1850/// Replace a failed factory-backed worker connection.
1851///
1852/// Returns `true` when a replacement was installed and the worker should
1853/// continue. All terminal outcomes fail requests queued during reconnect and
1854/// return `false`.
1855#[allow(clippy::too_many_arguments)]
1856async fn reconnect_worker_connection(
1857    conn: &mut RedisConnection,
1858    rx: &mut mpsc::Receiver<WorkerRequest>,
1859    factory: &dyn ConnectionFactory,
1860    reconnect: &ReconnectConfig,
1861    event_bus: Option<&ConnectionEventBus>,
1862    shutdown: &mut watch::Receiver<bool>,
1863    control: &WorkerControl,
1864    connection_health: &ConnectionHealthPublisher,
1865    outage_reported: &mut bool,
1866    maintenance_push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
1867    maintenance_enabled: bool,
1868) -> bool {
1869    let started = Instant::now();
1870    match reconnect_with_backoff(
1871        factory,
1872        reconnect,
1873        event_bus,
1874        shutdown,
1875        control,
1876        maintenance_enabled,
1877    )
1878    .await
1879    {
1880        ReconnectOutcome::Connected(new_conn, new_push_rx, attempts) => {
1881            let reconnected = control.with_active_handle(|| {
1882                *conn = *new_conn;
1883                *maintenance_push_rx = new_push_rx;
1884                // Publish health before the lifecycle event so observers can
1885                // rely on the event/state ordering in either direction.
1886                connection_health.set(true);
1887                *outage_reported = false;
1888                if let Some(events) = event_bus {
1889                    events.publish(ConnectionEvent::Reconnected {
1890                        attempts,
1891                        elapsed: started.elapsed(),
1892                    });
1893                }
1894            });
1895            if reconnected.is_some() {
1896                true
1897            } else {
1898                fail_queued_requests(rx);
1899                false
1900            }
1901        }
1902        ReconnectOutcome::Exhausted | ReconnectOutcome::Shutdown => {
1903            fail_queued_requests(rx);
1904            false
1905        }
1906    }
1907}
1908
1909fn fail_queued_requests(rx: &mut mpsc::Receiver<WorkerRequest>) {
1910    rx.close();
1911    while let Ok(request) = rx.try_recv() {
1912        request.fail(RedisError::ConnectionClosed);
1913    }
1914}
1915
1916fn publish_worker_outage(
1917    event_bus: Option<&ConnectionEventBus>,
1918    outage_reported: &mut bool,
1919    reason: ConnectionDisconnectReason,
1920    control: &WorkerControl,
1921) {
1922    if *outage_reported {
1923        return;
1924    }
1925    let active = control.with_active_handle(|| {
1926        if let Some(events) = event_bus {
1927            events.publish(ConnectionEvent::Disconnected { reason });
1928        }
1929    });
1930    if active.is_some() {
1931        *outage_reported = true;
1932    }
1933}
1934
1935/// True if `frame` is a `READONLY` error reply (write attempted against a
1936/// replica). Used to detect a demoted Sentinel master.
1937fn is_readonly_frame(frame: &Frame) -> bool {
1938    matches!(frame, Frame::Error(e) if e.len() >= 8 && e[..8].eq_ignore_ascii_case(b"READONLY"))
1939}
1940
1941/// Send a batch of requests as a pipeline and route responses back.
1942///
1943/// Frames from all requests are flattened into a single `execute_pipeline`
1944/// call in request order, preserving within-request contiguity: every frame
1945/// from a given `Multi` request appears consecutively on the wire, with no
1946/// other caller's frames in between. Responses are partitioned back to the
1947/// originating request.
1948///
1949/// Takes ownership of `batch` in a single pass to move frames directly into
1950/// the pipeline vec, avoiding per-Frame clones. A `Responder` enum tracks
1951/// response routing alongside the frame vec so both the success and error
1952/// paths can notify all senders without a second iteration.
1953///
1954/// Returns `Ok(())` on success and a typed [`PipelineFailure`] on connection
1955/// loss, timeout, or `READONLY` so the worker can publish the cause and decide
1956/// whether to reconnect. All individual response channels are always notified
1957/// before this returns.
1958async fn flush_batch(
1959    conn: &mut RedisConnection,
1960    batch: Vec<WorkerRequest>,
1961    response_timeout: Option<Duration>,
1962    recorder: Option<&Arc<dyn MetricsRecorder>>,
1963    reconnect_on_readonly: bool,
1964    connection_health: &ConnectionHealthPublisher,
1965) -> Result<(), PipelineFailure> {
1966    // A request can be accepted into the worker queue and then cancelled while
1967    // this worker waits out `batch_window`. `timeout_at` drops the response
1968    // receiver at the command deadline; prune those requests at the last
1969    // batching boundary before frames are moved into the wire pipeline.
1970    let mut batch = batch;
1971    batch.retain(|request| !request.response_is_closed());
1972    if batch.is_empty() {
1973        return Ok(());
1974    }
1975
1976    // Owned single-pass: move frames out of each request directly into the
1977    // pipeline vec, and collect response senders into a parallel `responders`
1978    // vec. This eliminates the per-Frame clone that the previous two-pass
1979    // implementation (borrow to build frames, then own to route responses)
1980    // required.
1981    let total_frames: usize = batch.iter().map(|r| r.frame_count()).sum();
1982
1983    // Report the batch size -- the one observability signal only the worker can
1984    // see (per-command metrics belong to the composed MetricsLayer instead).
1985    if let Some(recorder) = recorder {
1986        recorder.pipeline_flushed(total_frames);
1987    }
1988
1989    let mut frames: Vec<Frame> = Vec::with_capacity(total_frames);
1990
1991    enum Responder {
1992        Single(oneshot::Sender<Result<Frame, RedisError>>),
1993        Multi(usize, oneshot::Sender<Result<Vec<Frame>, RedisError>>),
1994    }
1995    let mut responders: Vec<Responder> = Vec::with_capacity(batch.len());
1996
1997    for req in batch {
1998        match req {
1999            WorkerRequest::Single { frame, response_tx } => {
2000                frames.push(frame);
2001                responders.push(Responder::Single(response_tx));
2002            }
2003            WorkerRequest::Multi {
2004                frames: fs,
2005                response_tx,
2006            } => {
2007                let count = fs.len();
2008                frames.extend(fs);
2009                responders.push(Responder::Multi(count, response_tx));
2010            }
2011        }
2012    }
2013
2014    let exec_result = match response_timeout {
2015        Some(timeout) => match tokio::time::timeout(timeout, conn.execute_pipeline(frames)).await {
2016            Ok(result) => result,
2017            Err(_elapsed) => {
2018                // Response timeout: the connection has written commands whose
2019                // replies were never read, so its state is now unknown. Fail
2020                // the whole batch and signal the worker (via Err) to discard
2021                // and reconnect the connection.
2022                connection_health.set(false);
2023                for responder in responders {
2024                    match responder {
2025                        Responder::Single(tx) => {
2026                            let _ = tx.send(Err(RedisError::CommandTimeout));
2027                        }
2028                        Responder::Multi(_, tx) => {
2029                            let _ = tx.send(Err(RedisError::CommandTimeout));
2030                        }
2031                    }
2032                }
2033                return Err(PipelineFailure::CommandTimeout);
2034            }
2035        },
2036        None => conn.execute_pipeline(frames).await,
2037    };
2038    match exec_result {
2039        Ok(responses) => {
2040            // A READONLY reply means the connection points at a replica (e.g. a
2041            // Sentinel master demoted via REPLICAOF). Detect it before routing
2042            // so the caller still gets the error, then signal the worker to
2043            // reconnect via the factory onto a real master.
2044            let saw_readonly = reconnect_on_readonly && responses.iter().any(is_readonly_frame);
2045
2046            if saw_readonly {
2047                connection_health.set(false);
2048            }
2049
2050            let mut iter = responses.into_iter();
2051            for responder in responders {
2052                match responder {
2053                    Responder::Single(tx) => {
2054                        let _ = tx.send(iter.next().ok_or(RedisError::ConnectionClosed));
2055                    }
2056                    Responder::Multi(count, tx) => {
2057                        let collected: Vec<Frame> = iter.by_ref().take(count).collect();
2058                        if collected.len() == count {
2059                            let _ = tx.send(Ok(collected));
2060                        } else {
2061                            let _ = tx.send(Err(RedisError::ConnectionClosed));
2062                        }
2063                    }
2064                }
2065            }
2066            if saw_readonly {
2067                Err(PipelineFailure::ReadOnly)
2068            } else {
2069                Ok(())
2070            }
2071        }
2072        Err(error) => {
2073            connection_health.set(false);
2074            for responder in responders {
2075                match responder {
2076                    Responder::Single(tx) => {
2077                        let _ = tx.send(Err(RedisError::ConnectionClosed));
2078                    }
2079                    Responder::Multi(_, tx) => {
2080                        let _ = tx.send(Err(RedisError::ConnectionClosed));
2081                    }
2082                }
2083            }
2084            Err(PipelineFailure::Connection(error))
2085        }
2086    }
2087}
2088
2089enum ReconnectOutcome {
2090    Connected(
2091        Box<RedisConnection>,
2092        Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
2093        usize,
2094    ),
2095    Exhausted,
2096    Shutdown,
2097}
2098
2099async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
2100    if *shutdown.borrow() {
2101        return;
2102    }
2103    let _ = shutdown.changed().await;
2104}
2105
2106/// Reconnect with exponential backoff until connected, exhausted, or the last
2107/// public service handle requests worker shutdown.
2108async fn reconnect_with_backoff(
2109    factory: &dyn ConnectionFactory,
2110    config: &ReconnectConfig,
2111    event_bus: Option<&ConnectionEventBus>,
2112    shutdown: &mut watch::Receiver<bool>,
2113    control: &WorkerControl,
2114    maintenance_enabled: bool,
2115) -> ReconnectOutcome {
2116    let mut attempt: usize = 0;
2117    loop {
2118        if *shutdown.borrow() {
2119            return ReconnectOutcome::Shutdown;
2120        }
2121        if config.attempt_exhausted(attempt) {
2122            warn!(
2123                attempts = attempt,
2124                "auto_pipeline: reconnect attempts exhausted"
2125            );
2126            let active = control.with_active_handle(|| {
2127                if let Some(events) = event_bus {
2128                    events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
2129                }
2130            });
2131            if active.is_none() {
2132                return ReconnectOutcome::Shutdown;
2133            }
2134            return ReconnectOutcome::Exhausted;
2135        }
2136        let delay = config.delay_for_attempt(attempt);
2137        let active = control.with_active_handle(|| {
2138            if let Some(events) = event_bus {
2139                events.publish(ConnectionEvent::ReconnectAttempt {
2140                    attempt: attempt + 1,
2141                    delay,
2142                });
2143            }
2144        });
2145        if active.is_none() {
2146            return ReconnectOutcome::Shutdown;
2147        }
2148        if !delay.is_zero() {
2149            tokio::select! {
2150                biased;
2151                () = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
2152                () = tokio::time::sleep(delay) => {}
2153            }
2154        }
2155        attempt += 1;
2156        let connect = connect_and_prepare(factory, config.connect_timeout, maintenance_enabled);
2157        tokio::pin!(connect);
2158        let result = tokio::select! {
2159            biased;
2160            () = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
2161            result = &mut connect => result,
2162        };
2163        match result {
2164            Ok((conn, push_rx)) => {
2165                return ReconnectOutcome::Connected(Box::new(conn), push_rx, attempt);
2166            }
2167            Err(e) => {
2168                warn!(attempt, error = %e, "auto_pipeline: reconnect attempt failed");
2169                let active = control.with_active_handle(|| {
2170                    if let Some(events) = event_bus {
2171                        events.publish_with(|| ConnectionEvent::ReconnectFailed {
2172                            attempt,
2173                            error: Arc::from(e.to_string()),
2174                        });
2175                    }
2176                });
2177                if active.is_none() {
2178                    return ReconnectOutcome::Shutdown;
2179                }
2180                continue;
2181            }
2182        }
2183    }
2184}
2185
2186impl Service<Frame> for AutoPipelineService {
2187    type Response = Frame;
2188    type Error = RedisError;
2189    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
2190
2191    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
2192        if self.lease.control.is_shutdown_requested() {
2193            let _ = self.poll_tx.abort_send();
2194            return Poll::Ready(Err(RedisError::ConnectionClosed));
2195        }
2196        // Register a per-clone wake before polling the queue. Recheck the
2197        // linearized state afterward so shutdown cannot race between the first
2198        // check and notification registration.
2199        let shutdown_poll = self.shutdown_notified.as_mut().poll(cx);
2200        if shutdown_poll.is_ready() || self.lease.control.is_shutdown_requested() {
2201            let _ = self.poll_tx.abort_send();
2202            return Poll::Ready(Err(RedisError::ConnectionClosed));
2203        }
2204        if self.shed_load {
2205            // Load-shedding: readiness only reflects channel liveness; `call`
2206            // does the (possibly failing) `try_send`.
2207            return if self.tx.is_closed() {
2208                Poll::Ready(Err(RedisError::ConnectionClosed))
2209            } else {
2210                Poll::Ready(Ok(()))
2211            };
2212        }
2213        // Back-pressure: reserve a queue slot, pending until one is free.
2214        self.poll_tx
2215            .poll_reserve(cx)
2216            .map_err(|_| RedisError::ConnectionClosed)
2217    }
2218
2219    fn call(&mut self, frame: Frame) -> Self::Future {
2220        let (resp_tx, resp_rx) = oneshot::channel();
2221        let request = WorkerRequest::Single {
2222            frame,
2223            response_tx: resp_tx,
2224        };
2225
2226        if self.shed_load {
2227            // Accept or reject load-shed work synchronously. A returned but
2228            // unpolled response future must not retain a hidden channel sender
2229            // that can keep the worker alive after the final service handle is
2230            // shut down.
2231            let send_result = self
2232                .lease
2233                .control
2234                .with_admission(|| {
2235                    self.tx.try_send(request).map_err(|e| match e {
2236                        tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
2237                        tokio::sync::mpsc::error::TrySendError::Closed(_) => {
2238                            RedisError::ConnectionClosed
2239                        }
2240                    })
2241                })
2242                .unwrap_or(Err(RedisError::ConnectionClosed));
2243            return Box::pin(async move {
2244                send_result?;
2245                resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
2246            });
2247        }
2248
2249        // Back-pressure: fill the slot reserved by `poll_ready`. Per the Tower
2250        // contract, `poll_ready` must have returned `Ready(Ok)` first.
2251        let send_result = self
2252            .lease
2253            .control
2254            .with_admission(|| {
2255                self.poll_tx
2256                    .send_item(request)
2257                    .map_err(|_| RedisError::ConnectionClosed)
2258            })
2259            .unwrap_or_else(|| {
2260                let _ = self.poll_tx.abort_send();
2261                Err(RedisError::ConnectionClosed)
2262            });
2263        Box::pin(async move {
2264            send_result?;
2265            resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
2266        })
2267    }
2268}
2269
2270impl Clone for AutoPipelineService {
2271    fn clone(&self) -> Self {
2272        // Build a fresh PollSender so the clone starts with no reservation held.
2273        Self {
2274            tx: self.tx.clone(),
2275            poll_tx: PollSender::new(self.tx.clone()),
2276            shed_load: self.shed_load,
2277            lease: self.lease.clone(),
2278            shutdown_notified: Box::pin(self.lease.control.shutdown_notified()),
2279            connection_health: self.connection_health.clone(),
2280            worker: Arc::clone(&self.worker),
2281        }
2282    }
2283}
2284
2285impl AutoPipelineService {
2286    /// Returns the current number of requests pending in the internal queue.
2287    ///
2288    /// This is an instantaneous snapshot; the value may change immediately
2289    /// after reading. Use it for observability (metrics, health checks).
2290    pub fn queue_depth(&self) -> usize {
2291        self.tx.max_capacity() - self.tx.capacity()
2292    }
2293}
2294
2295#[cfg(test)]
2296mod tests {
2297    use super::*;
2298
2299    #[test]
2300    fn lazy_factory_requires_an_entered_tokio_runtime() {
2301        let result = std::panic::catch_unwind(|| {
2302            AutoPipelineService::with_lazy_factory(
2303                || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) },
2304                AutoPipelineConfig::default(),
2305                AutoPipelineReconnectConfig::default(),
2306            )
2307        });
2308        assert!(result.is_err());
2309    }
2310
2311    #[tokio::test]
2312    async fn lazy_factory_defers_connection_and_retries_after_initial_failure() {
2313        use futures::{SinkExt, StreamExt};
2314        use std::sync::atomic::{AtomicUsize, Ordering};
2315        use tokio_util::codec::Framed;
2316
2317        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2318        let addr = listener.local_addr().unwrap();
2319        let server = tokio::spawn(async move {
2320            let (stream, _) = listener.accept().await.unwrap();
2321            let mut framed = Framed::new(
2322                redis_tower_core::RedisStream::Tcp(stream),
2323                redis_tower_core::RespCodec::new(),
2324            );
2325            let request = framed
2326                .next()
2327                .await
2328                .expect("client closed before request")
2329                .expect("client sent an invalid request");
2330            assert_eq!(
2331                request,
2332                Frame::SimpleString(bytes::Bytes::from_static(b"PING"))
2333            );
2334            framed
2335                .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
2336                .await
2337                .unwrap();
2338            let _ = framed.next().await;
2339        });
2340
2341        let calls = Arc::new(AtomicUsize::new(0));
2342        let factory_calls = Arc::clone(&calls);
2343        let factory = move || {
2344            let factory_calls = Arc::clone(&factory_calls);
2345            async move {
2346                if factory_calls.fetch_add(1, Ordering::AcqRel) == 0 {
2347                    return Err(RedisError::ConnectionClosed);
2348                }
2349                let stream = tokio::net::TcpStream::connect(addr)
2350                    .await
2351                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2352                Ok(RedisConnection::from_stream(
2353                    redis_tower_core::RedisStream::Tcp(stream),
2354                ))
2355            }
2356        };
2357
2358        let events = ConnectionEventBus::new(8);
2359        let mut event_stream = events.subscribe();
2360        let mut service = AutoPipelineService::with_lazy_factory_and_events(
2361            factory,
2362            AutoPipelineConfig::default(),
2363            AutoPipelineReconnectConfig::default(),
2364            events,
2365        );
2366
2367        assert_eq!(calls.load(Ordering::Acquire), 0);
2368        assert!(!service.is_connection_healthy());
2369        assert!(
2370            tokio::time::timeout(Duration::from_millis(10), event_stream.recv())
2371                .await
2372                .is_err(),
2373            "lazy construction must not publish a connection event"
2374        );
2375
2376        futures::future::poll_fn(|cx| service.poll_ready(cx))
2377            .await
2378            .unwrap();
2379        assert!(matches!(
2380            service
2381                .call(Frame::SimpleString(bytes::Bytes::from_static(b"FIRST")))
2382                .await,
2383            Err(RedisError::ConnectionClosed)
2384        ));
2385        assert!(!service.is_connection_healthy());
2386        assert!(matches!(
2387            event_stream.recv().await.unwrap(),
2388            ConnectionEvent::ConnectFailed { .. }
2389        ));
2390
2391        futures::future::poll_fn(|cx| service.poll_ready(cx))
2392            .await
2393            .unwrap();
2394        assert_eq!(
2395            service
2396                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
2397                .await
2398                .unwrap(),
2399            Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
2400        );
2401        assert!(service.is_connection_healthy());
2402        assert_eq!(
2403            event_stream.recv().await.unwrap(),
2404            ConnectionEvent::Connected
2405        );
2406        assert_eq!(calls.load(Ordering::Acquire), 2);
2407
2408        service.shutdown().await;
2409        server.await.unwrap();
2410    }
2411
2412    #[tokio::test]
2413    async fn lazy_factory_shutdown_cancels_first_connection_attempt() {
2414        use tokio::sync::Notify;
2415
2416        let entered = Arc::new(Notify::new());
2417        let factory_entered = Arc::clone(&entered);
2418        let factory = move || {
2419            let factory_entered = Arc::clone(&factory_entered);
2420            async move {
2421                factory_entered.notify_one();
2422                futures::future::pending::<Result<RedisConnection, RedisError>>().await
2423            }
2424        };
2425        let mut service = AutoPipelineService::with_lazy_factory(
2426            factory,
2427            AutoPipelineConfig {
2428                shed_load_on_full: true,
2429                ..AutoPipelineConfig::default()
2430            },
2431            AutoPipelineReconnectConfig::default(),
2432        );
2433
2434        let response = service.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")));
2435        entered.notified().await;
2436        tokio::time::timeout(Duration::from_secs(1), service.shutdown())
2437            .await
2438            .expect("shutdown must cancel a deferred connection attempt");
2439        assert!(matches!(response.await, Err(RedisError::ConnectionClosed)));
2440    }
2441
2442    #[tokio::test]
2443    async fn initial_factory_timeout_publishes_connect_failed() {
2444        let factory =
2445            || async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
2446        let events = ConnectionEventBus::new(4);
2447        let mut stream = events.subscribe();
2448        let reconnect = AutoPipelineReconnectConfig::new(
2449            ReconnectConfig::default().connect_timeout(Duration::from_millis(10)),
2450        );
2451
2452        let result = AutoPipelineService::with_factory_and_events(
2453            factory,
2454            AutoPipelineConfig::default(),
2455            reconnect,
2456            events,
2457        )
2458        .await;
2459        assert!(matches!(result, Err(RedisError::ConnectTimeout)));
2460        assert_eq!(
2461            stream.recv().await.unwrap(),
2462            ConnectionEvent::ConnectFailed {
2463                error: Arc::from(RedisError::ConnectTimeout.to_string()),
2464            }
2465        );
2466    }
2467
2468    #[tokio::test]
2469    async fn reconnect_factory_timeout_is_counted_and_exhausted() {
2470        let factory =
2471            || async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
2472        let config = ReconnectConfig {
2473            max_retries: Some(0),
2474            base_delay: Duration::ZERO,
2475            max_delay: Duration::ZERO,
2476            jitter: false,
2477            connect_timeout: Some(Duration::from_millis(10)),
2478        };
2479        let events = ConnectionEventBus::new(4);
2480        let mut stream = events.subscribe();
2481        let (shutdown_tx, mut shutdown) = watch::channel(false);
2482        let control = WorkerControl::new(shutdown_tx, None);
2483
2484        assert!(matches!(
2485            reconnect_with_backoff(
2486                &factory,
2487                &config,
2488                Some(&events),
2489                &mut shutdown,
2490                &control,
2491                false,
2492            )
2493            .await,
2494            ReconnectOutcome::Exhausted
2495        ));
2496        assert_eq!(
2497            stream.recv().await.unwrap(),
2498            ConnectionEvent::ReconnectAttempt {
2499                attempt: 1,
2500                delay: Duration::ZERO,
2501            }
2502        );
2503        assert_eq!(
2504            stream.recv().await.unwrap(),
2505            ConnectionEvent::ReconnectFailed {
2506                attempt: 1,
2507                error: Arc::from(RedisError::ConnectTimeout.to_string()),
2508            }
2509        );
2510        assert_eq!(
2511            stream.recv().await.unwrap(),
2512            ConnectionEvent::ReconnectExhausted { attempts: 1 }
2513        );
2514    }
2515
2516    #[tokio::test]
2517    async fn reconnect_exhaustion_is_published_after_final_failure() {
2518        let factory = || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) };
2519        let config = ReconnectConfig {
2520            max_retries: Some(2),
2521            base_delay: Duration::ZERO,
2522            max_delay: Duration::ZERO,
2523            jitter: false,
2524            connect_timeout: None,
2525        };
2526        let events = ConnectionEventBus::new(8);
2527        let mut stream = events.subscribe();
2528        let (shutdown_tx, mut shutdown) = watch::channel(false);
2529        let control = WorkerControl::new(shutdown_tx, None);
2530
2531        assert!(matches!(
2532            reconnect_with_backoff(
2533                &factory,
2534                &config,
2535                Some(&events),
2536                &mut shutdown,
2537                &control,
2538                false,
2539            )
2540            .await,
2541            ReconnectOutcome::Exhausted
2542        ));
2543
2544        assert_eq!(
2545            stream.recv().await.unwrap(),
2546            ConnectionEvent::ReconnectAttempt {
2547                attempt: 1,
2548                delay: Duration::ZERO,
2549            }
2550        );
2551        assert!(matches!(
2552            stream.recv().await.unwrap(),
2553            ConnectionEvent::ReconnectFailed { attempt: 1, .. }
2554        ));
2555        assert_eq!(
2556            stream.recv().await.unwrap(),
2557            ConnectionEvent::ReconnectAttempt {
2558                attempt: 2,
2559                delay: Duration::ZERO,
2560            }
2561        );
2562        assert!(matches!(
2563            stream.recv().await.unwrap(),
2564            ConnectionEvent::ReconnectFailed { attempt: 2, .. }
2565        ));
2566        assert_eq!(
2567            stream.recv().await.unwrap(),
2568            ConnectionEvent::ReconnectAttempt {
2569                attempt: 3,
2570                delay: Duration::ZERO,
2571            }
2572        );
2573        assert!(matches!(
2574            stream.recv().await.unwrap(),
2575            ConnectionEvent::ReconnectFailed { attempt: 3, .. }
2576        ));
2577        assert_eq!(
2578            stream.recv().await.unwrap(),
2579            ConnectionEvent::ReconnectExhausted { attempts: 3 }
2580        );
2581    }
2582
2583    #[tokio::test]
2584    async fn factory_worker_detects_idle_disconnect_and_reconnects_in_order() {
2585        use std::sync::atomic::{AtomicUsize, Ordering};
2586        use tokio::sync::{Notify, oneshot};
2587
2588        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2589        let addr = listener.local_addr().unwrap();
2590        let (first_closed_tx, first_closed_rx) = oneshot::channel();
2591        let server = tokio::spawn(async move {
2592            let (first, _) = listener.accept().await.unwrap();
2593            drop(first);
2594            let _ = first_closed_tx.send(());
2595
2596            let (second, _) = listener.accept().await.unwrap();
2597            let _second = second;
2598            futures::future::pending::<()>().await;
2599        });
2600
2601        let calls = Arc::new(AtomicUsize::new(0));
2602        let allow_reconnect = Arc::new(Notify::new());
2603        let factory_calls = Arc::clone(&calls);
2604        let factory_allow_reconnect = Arc::clone(&allow_reconnect);
2605        let factory = move || {
2606            let factory_calls = Arc::clone(&factory_calls);
2607            let factory_allow_reconnect = Arc::clone(&factory_allow_reconnect);
2608            async move {
2609                let call = factory_calls.fetch_add(1, Ordering::AcqRel);
2610                if call == 1 {
2611                    return Err(RedisError::ConnectionClosed);
2612                }
2613                if call == 2 {
2614                    factory_allow_reconnect.notified().await;
2615                }
2616                let stream = tokio::net::TcpStream::connect(addr)
2617                    .await
2618                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2619                Ok(RedisConnection::from_stream(
2620                    redis_tower_core::RedisStream::Tcp(stream),
2621                ))
2622            }
2623        };
2624
2625        let events = ConnectionEventBus::new(16);
2626        let mut event_stream = events.subscribe();
2627        let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2628            max_retries: Some(3),
2629            base_delay: Duration::ZERO,
2630            max_delay: Duration::ZERO,
2631            jitter: false,
2632            connect_timeout: None,
2633        });
2634        let service = AutoPipelineService::with_factory_and_events(
2635            factory,
2636            AutoPipelineConfig::default(),
2637            reconnect,
2638            events,
2639        )
2640        .await
2641        .unwrap();
2642        let mut health = service.subscribe_connection_health();
2643        assert_eq!(
2644            event_stream.recv().await.unwrap(),
2645            ConnectionEvent::Connected
2646        );
2647
2648        first_closed_rx.await.unwrap();
2649        assert!(matches!(
2650            event_stream.recv().await.unwrap(),
2651            ConnectionEvent::Disconnected {
2652                reason: ConnectionDisconnectReason::ConnectionError { .. }
2653            }
2654        ));
2655        assert_eq!(
2656            event_stream.recv().await.unwrap(),
2657            ConnectionEvent::ReconnectAttempt {
2658                attempt: 1,
2659                delay: Duration::ZERO,
2660            }
2661        );
2662        assert_eq!(
2663            event_stream.recv().await.unwrap(),
2664            ConnectionEvent::ReconnectFailed {
2665                attempt: 1,
2666                error: Arc::from("connection closed"),
2667            }
2668        );
2669        assert_eq!(
2670            event_stream.recv().await.unwrap(),
2671            ConnectionEvent::ReconnectAttempt {
2672                attempt: 2,
2673                delay: Duration::ZERO,
2674            }
2675        );
2676        health.changed().await.unwrap();
2677        assert!(!*health.borrow_and_update());
2678        assert!(!service.is_connection_healthy());
2679
2680        allow_reconnect.notify_one();
2681        assert!(matches!(
2682            event_stream.recv().await.unwrap(),
2683            ConnectionEvent::Reconnected { attempts: 2, .. }
2684        ));
2685        health.changed().await.unwrap();
2686        assert!(*health.borrow_and_update());
2687        assert!(service.is_connection_healthy());
2688        assert_eq!(calls.load(Ordering::Acquire), 3);
2689
2690        service.shutdown().await;
2691        server.abort();
2692    }
2693
2694    #[tokio::test]
2695    async fn shutdown_handle_cancels_hanging_unlimited_reconnect_with_clones() {
2696        use std::sync::atomic::{AtomicUsize, Ordering};
2697        use tokio::sync::{Notify, oneshot};
2698
2699        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2700        let addr = listener.local_addr().unwrap();
2701        let (first_closed_tx, first_closed_rx) = oneshot::channel();
2702        let server = tokio::spawn(async move {
2703            let (first, _) = listener.accept().await.unwrap();
2704            drop(first);
2705            let _ = first_closed_tx.send(());
2706        });
2707
2708        let calls = Arc::new(AtomicUsize::new(0));
2709        let reconnect_entered = Arc::new(Notify::new());
2710        let factory_calls = Arc::clone(&calls);
2711        let factory_entered = Arc::clone(&reconnect_entered);
2712        let factory = move || {
2713            let factory_calls = Arc::clone(&factory_calls);
2714            let factory_entered = Arc::clone(&factory_entered);
2715            async move {
2716                if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
2717                    factory_entered.notify_one();
2718                    return futures::future::pending::<Result<RedisConnection, RedisError>>().await;
2719                }
2720                let stream = tokio::net::TcpStream::connect(addr)
2721                    .await
2722                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2723                Ok(RedisConnection::from_stream(
2724                    redis_tower_core::RedisStream::Tcp(stream),
2725                ))
2726            }
2727        };
2728
2729        let events = ConnectionEventBus::new(8);
2730        let mut event_stream = events.subscribe();
2731        let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2732            max_retries: None,
2733            base_delay: Duration::ZERO,
2734            max_delay: Duration::from_secs(5),
2735            jitter: false,
2736            connect_timeout: None,
2737        });
2738        let service = AutoPipelineService::with_factory_and_events(
2739            factory,
2740            AutoPipelineConfig::default(),
2741            reconnect,
2742            events,
2743        )
2744        .await
2745        .unwrap();
2746        let mut surviving_clone = service.clone();
2747        let shutdown = service.shutdown_handle();
2748        assert_eq!(
2749            event_stream.recv().await.unwrap(),
2750            ConnectionEvent::Connected
2751        );
2752
2753        first_closed_rx.await.unwrap();
2754        assert!(matches!(
2755            event_stream.recv().await.unwrap(),
2756            ConnectionEvent::Disconnected {
2757                reason: ConnectionDisconnectReason::ConnectionError { .. }
2758            }
2759        ));
2760        assert_eq!(
2761            event_stream.recv().await.unwrap(),
2762            ConnectionEvent::ReconnectAttempt {
2763                attempt: 1,
2764                delay: Duration::ZERO,
2765            }
2766        );
2767        reconnect_entered.notified().await;
2768        futures::future::poll_fn(|cx| surviving_clone.poll_ready(cx))
2769            .await
2770            .unwrap();
2771
2772        tokio::time::timeout(Duration::from_secs(1), shutdown.shutdown())
2773            .await
2774            .expect("controlled shutdown must cancel a hanging reconnect factory");
2775        assert_eq!(
2776            event_stream.recv().await.unwrap(),
2777            ConnectionEvent::Disconnected {
2778                reason: ConnectionDisconnectReason::Shutdown,
2779            }
2780        );
2781        assert!(matches!(
2782            surviving_clone
2783                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
2784                .await,
2785            Err(RedisError::ConnectionClosed)
2786        ));
2787        assert!(shutdown.is_shutdown_requested());
2788        assert!(shutdown.is_finished());
2789        drop(surviving_clone);
2790        drop(service);
2791        drop(shutdown);
2792        assert_eq!(
2793            event_stream.recv().await.unwrap_err(),
2794            crate::reconnect::ConnectionEventRecvError::Closed
2795        );
2796        assert_eq!(calls.load(Ordering::Acquire), 2);
2797
2798        server.await.unwrap();
2799    }
2800
2801    #[tokio::test]
2802    async fn exhausted_worker_still_publishes_shutdown_on_final_handle() {
2803        use std::sync::atomic::{AtomicUsize, Ordering};
2804        use tokio::sync::oneshot;
2805
2806        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
2807        let addr = listener.local_addr().unwrap();
2808        let (first_closed_tx, first_closed_rx) = oneshot::channel();
2809        let server = tokio::spawn(async move {
2810            let (first, _) = listener.accept().await.unwrap();
2811            drop(first);
2812            let _ = first_closed_tx.send(());
2813        });
2814
2815        let calls = Arc::new(AtomicUsize::new(0));
2816        let factory_calls = Arc::clone(&calls);
2817        let factory = move || {
2818            let factory_calls = Arc::clone(&factory_calls);
2819            async move {
2820                if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
2821                    return Err(RedisError::ConnectionClosed);
2822                }
2823                let stream = tokio::net::TcpStream::connect(addr)
2824                    .await
2825                    .map_err(|error| RedisError::connection(addr.to_string(), error))?;
2826                Ok(RedisConnection::from_stream(
2827                    redis_tower_core::RedisStream::Tcp(stream),
2828                ))
2829            }
2830        };
2831
2832        let events = ConnectionEventBus::new(8);
2833        let mut event_stream = events.subscribe();
2834        let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
2835            max_retries: Some(0),
2836            base_delay: Duration::ZERO,
2837            max_delay: Duration::ZERO,
2838            jitter: false,
2839            connect_timeout: None,
2840        });
2841        let service = AutoPipelineService::with_factory_and_events(
2842            factory,
2843            AutoPipelineConfig::default(),
2844            reconnect,
2845            events,
2846        )
2847        .await
2848        .unwrap();
2849        assert_eq!(
2850            event_stream.recv().await.unwrap(),
2851            ConnectionEvent::Connected
2852        );
2853
2854        first_closed_rx.await.unwrap();
2855        assert!(matches!(
2856            event_stream.recv().await.unwrap(),
2857            ConnectionEvent::Disconnected {
2858                reason: ConnectionDisconnectReason::ConnectionError { .. }
2859            }
2860        ));
2861        assert!(matches!(
2862            event_stream.recv().await.unwrap(),
2863            ConnectionEvent::ReconnectAttempt { attempt: 1, .. }
2864        ));
2865        assert!(matches!(
2866            event_stream.recv().await.unwrap(),
2867            ConnectionEvent::ReconnectFailed { attempt: 1, .. }
2868        ));
2869        assert_eq!(
2870            event_stream.recv().await.unwrap(),
2871            ConnectionEvent::ReconnectExhausted { attempts: 1 }
2872        );
2873
2874        tokio::time::timeout(Duration::from_secs(1), service.shutdown())
2875            .await
2876            .expect("shutdown should join an already exhausted worker");
2877        assert_eq!(
2878            event_stream.recv().await.unwrap(),
2879            ConnectionEvent::Disconnected {
2880                reason: ConnectionDisconnectReason::Shutdown,
2881            }
2882        );
2883        assert_eq!(
2884            event_stream.recv().await.unwrap_err(),
2885            crate::reconnect::ConnectionEventRecvError::Closed
2886        );
2887        assert_eq!(calls.load(Ordering::Acquire), 2);
2888
2889        server.await.unwrap();
2890    }
2891
2892    #[test]
2893    fn config_defaults() {
2894        let config = AutoPipelineConfig::default();
2895        assert_eq!(config.max_batch_size, 100);
2896        assert_eq!(config.batch_window, Duration::ZERO);
2897        assert_eq!(config.queue_capacity, 1024);
2898        assert!(
2899            !config.shed_load_on_full,
2900            "back-pressure is the default; load shedding is opt-in"
2901        );
2902        assert!(
2903            config.response_timeout.is_none(),
2904            "no response deadline by default"
2905        );
2906        assert!(
2907            config.metrics_recorder.is_none(),
2908            "no metrics recorder by default"
2909        );
2910    }
2911
2912    #[test]
2913    fn config_custom() {
2914        let config = AutoPipelineConfig {
2915            max_batch_size: 50,
2916            batch_window: Duration::from_micros(500),
2917            queue_capacity: 512,
2918            shed_load_on_full: true,
2919            response_timeout: Some(Duration::from_millis(250)),
2920            metrics_recorder: None,
2921            reconnect_on_readonly: false,
2922        };
2923        assert_eq!(config.max_batch_size, 50);
2924        assert_eq!(config.batch_window, Duration::from_micros(500));
2925        assert_eq!(config.queue_capacity, 512);
2926        assert!(config.shed_load_on_full);
2927        assert_eq!(config.response_timeout, Some(Duration::from_millis(250)));
2928        assert!(!config.reconnect_on_readonly);
2929    }
2930
2931    #[cfg(unix)]
2932    #[tokio::test]
2933    async fn fixed_worker_detects_idle_close_and_closes_health_and_queue() {
2934        let (client, server) = tokio::net::UnixStream::pair().unwrap();
2935        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
2936        let events = ConnectionEventBus::new(4);
2937        let mut event_stream = events.subscribe();
2938        let mut service =
2939            AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
2940        let mut health = service.subscribe_connection_health();
2941
2942        assert!(service.is_connection_healthy());
2943        assert!(*health.borrow());
2944        assert_eq!(
2945            event_stream.recv().await.unwrap(),
2946            ConnectionEvent::Connected
2947        );
2948
2949        drop(server);
2950        let disconnected = tokio::time::timeout(Duration::from_secs(1), event_stream.recv())
2951            .await
2952            .expect("idle close was not detected")
2953            .unwrap();
2954        assert!(matches!(
2955            disconnected,
2956            ConnectionEvent::Disconnected {
2957                reason: ConnectionDisconnectReason::ConnectionError { .. }
2958            }
2959        ));
2960        assert!(
2961            !service.is_connection_healthy(),
2962            "health must be false before Disconnected is observable"
2963        );
2964
2965        health
2966            .changed()
2967            .await
2968            .expect("the final unhealthy snapshot must be observable");
2969        assert!(!*health.borrow_and_update());
2970        assert!(
2971            health.changed().await.is_err(),
2972            "terminal fixed-worker loss must close the health channel"
2973        );
2974        assert!(
2975            futures::future::poll_fn(|cx| service.poll_ready(cx))
2976                .await
2977                .is_err(),
2978            "terminal idle loss must close the request queue"
2979        );
2980
2981        service.shutdown().await;
2982    }
2983
2984    #[cfg(unix)]
2985    #[tokio::test]
2986    async fn pipeline_failure_publishes_unhealthy_before_failing_responder() {
2987        use futures::StreamExt;
2988        use tokio_util::codec::Framed;
2989
2990        let (client, server) = tokio::net::UnixStream::pair().unwrap();
2991        let server = tokio::spawn(async move {
2992            let mut framed = Framed::new(
2993                redis_tower_core::RedisStream::Unix(server),
2994                redis_tower_core::RespCodec::new(),
2995            );
2996            framed
2997                .next()
2998                .await
2999                .expect("client closed before dispatch")
3000                .expect("client sent an invalid request");
3001            // Closing after reading the request makes this an in-flight
3002            // transport failure rather than an idle-close race.
3003        });
3004
3005        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3006        let events = ConnectionEventBus::new(4);
3007        let mut event_stream = events.subscribe();
3008        let mut service =
3009            AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
3010        assert_eq!(
3011            event_stream.recv().await.unwrap(),
3012            ConnectionEvent::Connected
3013        );
3014
3015        futures::future::poll_fn(|cx| service.poll_ready(cx))
3016            .await
3017            .unwrap();
3018        assert!(
3019            service
3020                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3021                .await
3022                .is_err()
3023        );
3024        assert!(
3025            !service.is_connection_healthy(),
3026            "the response future observed failure before shared health changed"
3027        );
3028        assert!(matches!(
3029            event_stream.recv().await.unwrap(),
3030            ConnectionEvent::Disconnected {
3031                reason: ConnectionDisconnectReason::ConnectionError { .. }
3032            }
3033        ));
3034
3035        service.shutdown().await;
3036        server.await.unwrap();
3037    }
3038
3039    #[cfg(unix)]
3040    #[tokio::test]
3041    async fn response_timeout_publishes_unhealthy_before_failing_responder() {
3042        use futures::StreamExt;
3043        use tokio_util::codec::Framed;
3044
3045        let (client, server) = tokio::net::UnixStream::pair().unwrap();
3046        let server = tokio::spawn(async move {
3047            let mut framed = Framed::new(
3048                redis_tower_core::RedisStream::Unix(server),
3049                redis_tower_core::RespCodec::new(),
3050            );
3051            framed
3052                .next()
3053                .await
3054                .expect("client closed before dispatch")
3055                .expect("client sent an invalid request");
3056            // Deliberately withhold the response until the client's deadline
3057            // quarantines and closes the transport.
3058            let _ = framed.next().await;
3059        });
3060
3061        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3062        let events = ConnectionEventBus::new(4);
3063        let mut event_stream = events.subscribe();
3064        let config = AutoPipelineConfig {
3065            response_timeout: Some(Duration::from_millis(10)),
3066            ..AutoPipelineConfig::default()
3067        };
3068        let mut service = AutoPipelineService::new_with_events(conn, config, events);
3069        assert_eq!(
3070            event_stream.recv().await.unwrap(),
3071            ConnectionEvent::Connected
3072        );
3073
3074        futures::future::poll_fn(|cx| service.poll_ready(cx))
3075            .await
3076            .unwrap();
3077        assert!(matches!(
3078            service
3079                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3080                .await,
3081            Err(RedisError::CommandTimeout)
3082        ));
3083        assert!(
3084            !service.is_connection_healthy(),
3085            "the timeout responder was notified before shared health changed"
3086        );
3087        assert_eq!(
3088            event_stream.recv().await.unwrap(),
3089            ConnectionEvent::Disconnected {
3090                reason: ConnectionDisconnectReason::CommandTimeout,
3091            }
3092        );
3093
3094        service.shutdown().await;
3095        server.await.unwrap();
3096    }
3097
3098    #[cfg(unix)]
3099    #[tokio::test]
3100    async fn idle_push_is_routed_without_delaying_the_next_request() {
3101        use futures::{SinkExt, StreamExt};
3102        use tokio_util::codec::Framed;
3103
3104        let (client, server) = tokio::net::UnixStream::pair().unwrap();
3105        let mut conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3106        let mut pushes = conn.subscribe_pushes();
3107        let expected_push = Frame::Push(vec![
3108            Frame::SimpleString(bytes::Bytes::from_static(b"invalidate")),
3109            Frame::Array(Some(vec![Frame::SimpleString(bytes::Bytes::from_static(
3110                b"key",
3111            ))])),
3112        ]);
3113        let server_push = expected_push.clone();
3114        let server = tokio::spawn(async move {
3115            let mut framed = Framed::new(
3116                redis_tower_core::RedisStream::Unix(server),
3117                redis_tower_core::RespCodec::new(),
3118            );
3119            framed.send(server_push).await.unwrap();
3120            framed
3121                .next()
3122                .await
3123                .expect("client closed before request")
3124                .expect("client sent an invalid request");
3125            framed
3126                .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3127                .await
3128                .unwrap();
3129            // Keep the socket open until the worker's graceful shutdown drops
3130            // its side, proving the health assertion below is not racy.
3131            let _ = framed.next().await;
3132        });
3133
3134        let mut service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
3135        assert_eq!(
3136            tokio::time::timeout(Duration::from_secs(1), pushes.recv())
3137                .await
3138                .expect("idle worker did not route the push")
3139                .unwrap(),
3140            expected_push
3141        );
3142        futures::future::poll_fn(|cx| service.poll_ready(cx))
3143            .await
3144            .unwrap();
3145        assert_eq!(
3146            service
3147                .call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
3148                .await
3149                .unwrap(),
3150            Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3151        );
3152        assert!(service.is_connection_healthy());
3153
3154        service.shutdown().await;
3155        server.await.unwrap();
3156    }
3157
3158    #[cfg(unix)]
3159    #[tokio::test]
3160    async fn cancelled_single_and_multi_requests_are_pruned_before_wire() {
3161        use tokio::io::AsyncReadExt;
3162
3163        let (client, mut server) = tokio::net::UnixStream::pair().unwrap();
3164        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
3165        let batch_window = Duration::from_millis(200);
3166        let mut service = AutoPipelineService::new(
3167            conn,
3168            AutoPipelineConfig {
3169                batch_window,
3170                ..AutoPipelineConfig::default()
3171            },
3172        );
3173
3174        futures::future::poll_fn(|cx| service.poll_ready(cx))
3175            .await
3176            .unwrap();
3177        let single = service.call(Frame::SimpleString(bytes::Bytes::from_static(
3178            b"CANCELLED-SINGLE",
3179        )));
3180        assert!(
3181            tokio::time::timeout(Duration::from_millis(25), single)
3182                .await
3183                .is_err(),
3184            "single request unexpectedly completed without a server response"
3185        );
3186
3187        let multi = service.call_pipeline(vec![
3188            Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-1")),
3189            Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-2")),
3190        ]);
3191        assert!(
3192            tokio::time::timeout(Duration::from_millis(25), multi)
3193                .await
3194                .is_err(),
3195            "multi request unexpectedly completed without a server response"
3196        );
3197
3198        // Both response receivers are closed well before the batch window
3199        // ends. The worker must drop both requests rather than flushing their
3200        // frames after the caller deadlines have expired.
3201        let mut bytes = [0u8; 256];
3202        assert!(
3203            tokio::time::timeout(
3204                batch_window + Duration::from_millis(100),
3205                server.read(&mut bytes)
3206            )
3207            .await
3208            .is_err(),
3209            "a cancelled queued request reached the Redis socket"
3210        );
3211
3212        service.shutdown().await;
3213    }
3214
3215    #[tokio::test]
3216    async fn lost_non_idempotent_reply_is_unknown_and_not_replayed_after_reconnect() {
3217        use futures::{SinkExt, StreamExt};
3218        use std::sync::atomic::{AtomicUsize, Ordering};
3219        use tokio_util::codec::Framed;
3220
3221        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3222        let addr = listener.local_addr().unwrap();
3223        let executions = Arc::new(AtomicUsize::new(0));
3224        let server_executions = Arc::clone(&executions);
3225        let server = tokio::spawn(async move {
3226            let (first, _) = listener.accept().await.unwrap();
3227            let mut first = Framed::new(
3228                redis_tower_core::RedisStream::Tcp(first),
3229                redis_tower_core::RespCodec::new(),
3230            );
3231            let request = first
3232                .next()
3233                .await
3234                .expect("client closed before dispatch")
3235                .expect("client sent an invalid request");
3236            assert_eq!(
3237                request,
3238                Frame::SimpleString(bytes::Bytes::from_static(b"INCR-NON-IDEMPOTENT"))
3239            );
3240            server_executions.fetch_add(1, Ordering::SeqCst);
3241            // The write executed, but its reply is lost. Closing here makes
3242            // execution unknowable to the client rather than proving failure.
3243            drop(first);
3244
3245            let (second, _) = listener.accept().await.unwrap();
3246            let mut second = Framed::new(
3247                redis_tower_core::RedisStream::Tcp(second),
3248                redis_tower_core::RespCodec::new(),
3249            );
3250            let request = second
3251                .next()
3252                .await
3253                .expect("client closed before post-reconnect probe")
3254                .expect("client sent an invalid post-reconnect request");
3255            assert_eq!(
3256                request,
3257                Frame::SimpleString(bytes::Bytes::from_static(b"READ-EXECUTION-COUNT")),
3258                "the lost-reply INCR was replayed on the replacement connection"
3259            );
3260            second
3261                .send(Frame::Integer(
3262                    server_executions.load(Ordering::SeqCst) as i64
3263                ))
3264                .await
3265                .unwrap();
3266            let _ = second.next().await;
3267        });
3268
3269        let factory = move || async move {
3270            let stream = tokio::net::TcpStream::connect(addr)
3271                .await
3272                .map_err(|error| RedisError::connection(addr.to_string(), error))?;
3273            Ok(RedisConnection::from_stream(
3274                redis_tower_core::RedisStream::Tcp(stream),
3275            ))
3276        };
3277        let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
3278            max_retries: Some(3),
3279            base_delay: Duration::ZERO,
3280            max_delay: Duration::ZERO,
3281            jitter: false,
3282            connect_timeout: None,
3283        });
3284        let mut service =
3285            AutoPipelineService::with_factory(factory, AutoPipelineConfig::default(), reconnect)
3286                .await
3287                .unwrap();
3288
3289        futures::future::poll_fn(|cx| service.poll_ready(cx))
3290            .await
3291            .unwrap();
3292        let lost = tokio::time::timeout(
3293            Duration::from_secs(1),
3294            service.call(Frame::SimpleString(bytes::Bytes::from_static(
3295                b"INCR-NON-IDEMPOTENT",
3296            ))),
3297        )
3298        .await
3299        .expect("lost-reply command did not complete");
3300        assert!(lost.is_err(), "a dropped reply was reported as success");
3301
3302        futures::future::poll_fn(|cx| service.poll_ready(cx))
3303            .await
3304            .unwrap();
3305        let count = tokio::time::timeout(
3306            Duration::from_secs(1),
3307            service.call(Frame::SimpleString(bytes::Bytes::from_static(
3308                b"READ-EXECUTION-COUNT",
3309            ))),
3310        )
3311        .await
3312        .expect("post-reconnect probe timed out")
3313        .unwrap();
3314        assert_eq!(count, Frame::Integer(1));
3315        assert_eq!(executions.load(Ordering::SeqCst), 1);
3316
3317        service.shutdown().await;
3318        server.await.unwrap();
3319    }
3320
3321    #[test]
3322    fn is_readonly_frame_detects_readonly_errors_only() {
3323        use bytes::Bytes;
3324        assert!(is_readonly_frame(&Frame::Error(Bytes::from(
3325            "READONLY You can't write against a read only replica."
3326        ))));
3327        // Case-insensitive on the prefix.
3328        assert!(is_readonly_frame(&Frame::Error(Bytes::from(
3329            "readonly nope"
3330        ))));
3331        // Other errors and non-error frames are not READONLY.
3332        assert!(!is_readonly_frame(&Frame::Error(Bytes::from(
3333            "WRONGTYPE Operation against a key"
3334        ))));
3335        assert!(!is_readonly_frame(&Frame::Error(Bytes::from("READ"))));
3336        assert!(!is_readonly_frame(&Frame::SimpleString(Bytes::from("OK"))));
3337    }
3338
3339    fn make_test_svc(
3340        tx: mpsc::Sender<WorkerRequest>,
3341        handle: tokio::task::JoinHandle<()>,
3342        shed_load: bool,
3343    ) -> AutoPipelineService {
3344        let (shutdown_tx, _shutdown_rx) = watch::channel(false);
3345        let (_connection_health_tx, connection_health) = watch::channel(true);
3346        let control = Arc::new(WorkerControl::new(shutdown_tx, None));
3347        AutoPipelineService {
3348            poll_tx: PollSender::new(tx.clone()),
3349            tx,
3350            shed_load,
3351            shutdown_notified: Box::pin(control.shutdown_notified()),
3352            lease: WorkerLease { control },
3353            connection_health,
3354            worker: Arc::new(WorkerHandle::new(handle)),
3355        }
3356    }
3357
3358    #[tokio::test]
3359    async fn closed_channel_error_is_retryable() {
3360        // When the background worker is gone (connection death), the error
3361        // must be retryable so upstream retry layers can reconnect.
3362        let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3363        drop(rx);
3364        let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3365
3366        let frame = Frame::SimpleString(b"PING"[..].into());
3367        let err = svc.call(frame).await.unwrap_err();
3368        assert!(err.is_retryable());
3369    }
3370
3371    #[tokio::test]
3372    async fn closed_channel_returns_error() {
3373        // Create a service with a channel that we immediately close.
3374        let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3375        drop(rx); // close the receiver
3376        let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3377
3378        // poll_ready should report closed.
3379        let ready = futures::future::poll_fn(|cx| svc.poll_ready(cx)).await;
3380        assert!(ready.is_err());
3381
3382        // call should also fail.
3383        let frame = Frame::SimpleString(b"PING"[..].into());
3384        let result = svc.call(frame).await;
3385        assert!(result.is_err());
3386    }
3387
3388    #[tokio::test]
3389    async fn shutdown_drains_an_unpolled_shed_load_call() {
3390        use futures::{SinkExt, StreamExt};
3391        use tokio_util::codec::Framed;
3392
3393        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3394        let addr = listener.local_addr().unwrap();
3395        let request = Frame::Array(Some(vec![Frame::BulkString(Some(
3396            bytes::Bytes::from_static(b"PING"),
3397        ))]));
3398        let expected_request = request.clone();
3399        let server = tokio::spawn(async move {
3400            let (stream, _) = listener.accept().await.unwrap();
3401            let mut framed = Framed::new(
3402                redis_tower_core::RedisStream::Tcp(stream),
3403                redis_tower_core::RespCodec::new(),
3404            );
3405            assert_eq!(framed.next().await.unwrap().unwrap(), expected_request);
3406            framed
3407                .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3408                .await
3409                .unwrap();
3410        });
3411
3412        let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3413        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3414        let events = ConnectionEventBus::new(4);
3415        let mut event_stream = events.subscribe();
3416        let config = AutoPipelineConfig {
3417            // A long window makes it deterministic that final shutdown, not
3418            // the timer, causes the already-accepted request to flush.
3419            batch_window: Duration::from_secs(30),
3420            shed_load_on_full: true,
3421            ..AutoPipelineConfig::default()
3422        };
3423        let mut service = AutoPipelineService::new_with_events(conn, config, events);
3424        assert_eq!(
3425            event_stream.recv().await.unwrap(),
3426            ConnectionEvent::Connected
3427        );
3428
3429        // Intentionally retain this future without polling it. `call` must
3430        // have accepted the request synchronously without hiding a sender in
3431        // the response future, and shutdown must drain that queued request.
3432        let response = service.call(request);
3433        tokio::time::timeout(Duration::from_secs(1), service.shutdown())
3434            .await
3435            .expect("an unpolled shed-load call must not keep shutdown alive");
3436        assert_eq!(
3437            response.await.unwrap(),
3438            Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3439        );
3440        assert_eq!(
3441            event_stream.recv().await.unwrap(),
3442            ConnectionEvent::Disconnected {
3443                reason: ConnectionDisconnectReason::Shutdown,
3444            }
3445        );
3446        assert_eq!(
3447            event_stream.recv().await.unwrap_err(),
3448            crate::reconnect::ConnectionEventRecvError::Closed
3449        );
3450
3451        server.await.unwrap();
3452    }
3453
3454    #[tokio::test]
3455    async fn shutdown_handle_drains_trait_erased_clones_and_is_idempotent() {
3456        use futures::{SinkExt, StreamExt};
3457        use tokio::sync::oneshot;
3458        use tokio_util::codec::Framed;
3459        use tower::util::BoxCloneService;
3460
3461        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3462        let addr = listener.local_addr().unwrap();
3463        let request = Frame::SimpleString(bytes::Bytes::from_static(b"PING"));
3464        let expected_request = request.clone();
3465        let (request_seen_tx, request_seen_rx) = oneshot::channel();
3466        let (send_reply_tx, send_reply_rx) = oneshot::channel();
3467        let server = tokio::spawn(async move {
3468            let (stream, _) = listener.accept().await.unwrap();
3469            let mut framed = Framed::new(
3470                redis_tower_core::RedisStream::Tcp(stream),
3471                redis_tower_core::RespCodec::new(),
3472            );
3473            assert_eq!(framed.next().await.unwrap().unwrap(), expected_request);
3474            let _ = request_seen_tx.send(());
3475            send_reply_rx.await.unwrap();
3476            framed
3477                .send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
3478                .await
3479                .unwrap();
3480            assert!(framed.next().await.is_none());
3481        });
3482
3483        let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3484        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3485        let events = ConnectionEventBus::new(4);
3486        let mut event_stream = events.subscribe();
3487        let service = AutoPipelineService::new_with_events(
3488            conn,
3489            AutoPipelineConfig {
3490                batch_window: Duration::from_secs(30),
3491                ..AutoPipelineConfig::default()
3492            },
3493            events,
3494        );
3495        assert_eq!(
3496            event_stream.recv().await.unwrap(),
3497            ConnectionEvent::Connected
3498        );
3499
3500        let shutdown = service.shutdown_handle();
3501        let mut erased: BoxCloneService<Frame, Frame, RedisError> = BoxCloneService::new(service);
3502        let mut surviving_clone = erased.clone();
3503        let mut idle_reserved_clone = erased.clone();
3504        futures::future::poll_fn(|cx| erased.poll_ready(cx))
3505            .await
3506            .unwrap();
3507        let response = erased.call(request);
3508        futures::future::poll_fn(|cx| idle_reserved_clone.poll_ready(cx))
3509            .await
3510            .unwrap();
3511
3512        let first_shutdown = shutdown.clone();
3513        let second_shutdown = shutdown.clone();
3514        let shutdown_task = tokio::spawn(async move {
3515            tokio::join!(first_shutdown.shutdown(), second_shutdown.shutdown());
3516        });
3517        tokio::time::timeout(Duration::from_secs(1), request_seen_rx)
3518            .await
3519            .expect("controlled shutdown did not drain the accepted request")
3520            .unwrap();
3521        send_reply_tx.send(()).unwrap();
3522        tokio::time::timeout(Duration::from_secs(1), shutdown_task)
3523            .await
3524            .expect("concurrent shutdown handles did not join the worker")
3525            .unwrap();
3526
3527        assert_eq!(
3528            response.await.unwrap(),
3529            Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
3530        );
3531        assert!(shutdown.is_shutdown_requested());
3532        assert!(shutdown.is_finished());
3533        assert!(
3534            futures::future::poll_fn(|cx| surviving_clone.poll_ready(cx))
3535                .await
3536                .is_err()
3537        );
3538        assert!(matches!(
3539            surviving_clone.call(Frame::Integer(1)).await,
3540            Err(RedisError::ConnectionClosed)
3541        ));
3542        assert!(matches!(
3543            idle_reserved_clone.call(Frame::Integer(2)).await,
3544            Err(RedisError::ConnectionClosed)
3545        ));
3546        assert_eq!(
3547            event_stream.recv().await.unwrap(),
3548            ConnectionEvent::Disconnected {
3549                reason: ConnectionDisconnectReason::Shutdown,
3550            }
3551        );
3552
3553        drop(erased);
3554        drop(surviving_clone);
3555        drop(idle_reserved_clone);
3556        drop(shutdown);
3557        assert_eq!(
3558            event_stream.recv().await.unwrap_err(),
3559            crate::reconnect::ConnectionEventRecvError::Closed
3560        );
3561        server.await.unwrap();
3562    }
3563
3564    #[tokio::test]
3565    async fn shutdown_handle_wakes_blocked_backpressure_before_drain_completes() {
3566        use futures::{SinkExt, StreamExt};
3567        use tokio::sync::oneshot;
3568        use tokio_util::codec::Framed;
3569
3570        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3571        let addr = listener.local_addr().unwrap();
3572        let first_request = Frame::SimpleString(bytes::Bytes::from_static(b"FIRST"));
3573        let second_request = Frame::SimpleString(bytes::Bytes::from_static(b"SECOND"));
3574        let (first_seen_tx, first_seen_rx) = oneshot::channel();
3575        let (release_first_tx, release_first_rx) = oneshot::channel();
3576        let server = tokio::spawn({
3577            let first_request = first_request.clone();
3578            let second_request = second_request.clone();
3579            async move {
3580                let (stream, _) = listener.accept().await.unwrap();
3581                let mut framed = Framed::new(
3582                    redis_tower_core::RedisStream::Tcp(stream),
3583                    redis_tower_core::RespCodec::new(),
3584                );
3585                assert_eq!(framed.next().await.unwrap().unwrap(), first_request);
3586                let _ = first_seen_tx.send(());
3587                release_first_rx.await.unwrap();
3588                framed.send(Frame::Integer(1)).await.unwrap();
3589                assert_eq!(framed.next().await.unwrap().unwrap(), second_request);
3590                framed.send(Frame::Integer(2)).await.unwrap();
3591                assert!(framed.next().await.is_none());
3592            }
3593        });
3594
3595        let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3596        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3597        let mut first = AutoPipelineService::new(
3598            conn,
3599            AutoPipelineConfig {
3600                queue_capacity: 1,
3601                max_batch_size: 1,
3602                batch_window: Duration::ZERO,
3603                ..AutoPipelineConfig::default()
3604            },
3605        );
3606        let shutdown = first.shutdown_handle();
3607
3608        futures::future::poll_fn(|cx| first.poll_ready(cx))
3609            .await
3610            .unwrap();
3611        let first_response = first.call(first_request);
3612        first_seen_rx.await.unwrap();
3613
3614        let mut second = first.clone();
3615        futures::future::poll_fn(|cx| second.poll_ready(cx))
3616            .await
3617            .unwrap();
3618        let second_response = second.call(second_request);
3619
3620        let mut pending_ready_service = first.clone();
3621        let (ready_pending_tx, ready_pending_rx) = oneshot::channel();
3622        let pending_ready_task = tokio::spawn(async move {
3623            let mut ready_pending_tx = Some(ready_pending_tx);
3624            futures::future::poll_fn(|cx| {
3625                let poll = pending_ready_service.poll_ready(cx);
3626                if poll.is_pending()
3627                    && let Some(ready_pending_tx) = ready_pending_tx.take()
3628                {
3629                    let _ = ready_pending_tx.send(());
3630                }
3631                poll
3632            })
3633            .await
3634        });
3635
3636        let mut pending_pipeline_service = first.clone();
3637        let (pipeline_pending_tx, pipeline_pending_rx) = oneshot::channel();
3638        let pending_pipeline_task = tokio::spawn(async move {
3639            let pending_pipeline = pending_pipeline_service.call_pipeline(vec![Frame::Integer(3)]);
3640            tokio::pin!(pending_pipeline);
3641            let mut pipeline_pending_tx = Some(pipeline_pending_tx);
3642            futures::future::poll_fn(|cx| {
3643                let poll = pending_pipeline.as_mut().poll(cx);
3644                if poll.is_pending()
3645                    && let Some(pipeline_pending_tx) = pipeline_pending_tx.take()
3646                {
3647                    let _ = pipeline_pending_tx.send(());
3648                }
3649                poll
3650            })
3651            .await
3652        });
3653
3654        ready_pending_rx
3655            .await
3656            .expect("poll_ready completed before the queue was released");
3657        pipeline_pending_rx
3658            .await
3659            .expect("call_pipeline completed before the queue was released");
3660
3661        let shutdown_task = tokio::spawn({
3662            let shutdown = shutdown.clone();
3663            async move { shutdown.shutdown().await }
3664        });
3665        while !shutdown.is_shutdown_requested() {
3666            tokio::task::yield_now().await;
3667        }
3668
3669        assert!(matches!(
3670            tokio::time::timeout(Duration::from_secs(1), pending_ready_task)
3671                .await
3672                .expect("pending poll_ready was not woken by shutdown")
3673                .unwrap(),
3674            Err(RedisError::ConnectionClosed)
3675        ));
3676        assert!(matches!(
3677            tokio::time::timeout(Duration::from_secs(1), pending_pipeline_task)
3678                .await
3679                .expect("pending call_pipeline was not woken by shutdown")
3680                .unwrap(),
3681            Err(RedisError::ConnectionClosed)
3682        ));
3683
3684        let mut new_pipeline_service = first.clone();
3685        assert!(matches!(
3686            tokio::time::timeout(
3687                Duration::from_secs(1),
3688                new_pipeline_service.call_pipeline(vec![Frame::Integer(4)]),
3689            )
3690            .await
3691            .expect("a new call_pipeline blocked after shutdown"),
3692            Err(RedisError::ConnectionClosed)
3693        ));
3694        assert!(
3695            !shutdown_task.is_finished(),
3696            "shutdown returned before accepted work drained"
3697        );
3698
3699        release_first_tx.send(()).unwrap();
3700        tokio::time::timeout(Duration::from_secs(1), shutdown_task)
3701            .await
3702            .expect("shutdown did not finish after accepted work drained")
3703            .unwrap();
3704        assert_eq!(first_response.await.unwrap(), Frame::Integer(1));
3705        assert_eq!(second_response.await.unwrap(), Frame::Integer(2));
3706
3707        drop(first);
3708        drop(second);
3709        drop(new_pipeline_service);
3710        drop(shutdown);
3711        server.await.unwrap();
3712    }
3713
3714    #[tokio::test]
3715    async fn final_shutdown_closes_an_idle_worker_with_a_retained_sender() {
3716        use tokio::io::AsyncReadExt;
3717
3718        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3719        let addr = listener.local_addr().unwrap();
3720        let server = tokio::spawn(async move {
3721            let (mut stream, _) = listener.accept().await.unwrap();
3722            let mut byte = [0_u8; 1];
3723            assert_eq!(stream.read(&mut byte).await.unwrap(), 0);
3724        });
3725
3726        let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3727        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3728        let service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
3729
3730        // Model a sender retained by an already-returned future or adapter.
3731        // Final public-handle shutdown must wake the idle worker independently
3732        // of sender count and close the receiver against any late work.
3733        let retained_sender = service.tx.clone();
3734        tokio::time::timeout(Duration::from_secs(1), service.shutdown())
3735            .await
3736            .expect("final shutdown must wake an idle worker");
3737        assert!(retained_sender.is_closed());
3738        drop(retained_sender);
3739
3740        tokio::time::timeout(Duration::from_secs(1), server)
3741            .await
3742            .expect("worker shutdown must close its Redis connection")
3743            .unwrap();
3744    }
3745
3746    #[tokio::test]
3747    async fn shutdown_last_clone_awaits_worker() {
3748        // Create a service whose worker exits immediately (no connection, no
3749        // requests -- the channel closes as soon as tx is dropped).
3750        let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3751        let handle = tokio::spawn(async move {
3752            // Simulate a worker that drains and exits when the channel closes.
3753            let mut rx = rx;
3754            while rx.recv().await.is_some() {}
3755        });
3756        let svc = make_test_svc(tx, handle, false);
3757
3758        // shutdown() on the sole instance should drop tx, succeed in
3759        // Arc::try_unwrap, and await the worker to completion.
3760        svc.shutdown().await;
3761        // If we reach here the worker has exited cleanly.
3762    }
3763
3764    #[tokio::test]
3765    async fn shutdown_publishes_one_intentional_disconnect() {
3766        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3767        let addr = listener.local_addr().unwrap();
3768        let server = tokio::spawn(async move {
3769            let (stream, _) = listener.accept().await.unwrap();
3770            let _stream = stream;
3771            futures::future::pending::<()>().await;
3772        });
3773        let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
3774        let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
3775        let events = ConnectionEventBus::new(4);
3776        let mut event_stream = events.subscribe();
3777        let service =
3778            AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
3779        assert_eq!(
3780            event_stream.recv().await.unwrap(),
3781            ConnectionEvent::Connected
3782        );
3783
3784        service.shutdown().await;
3785        assert_eq!(
3786            event_stream.recv().await.unwrap(),
3787            ConnectionEvent::Disconnected {
3788                reason: ConnectionDisconnectReason::Shutdown,
3789            }
3790        );
3791
3792        server.abort();
3793    }
3794
3795    #[tokio::test]
3796    async fn shutdown_non_last_clone_returns_immediately() {
3797        // When another clone is alive, Arc::try_unwrap fails and shutdown()
3798        // returns immediately without awaiting the worker.
3799        let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3800        // Spawn a worker that never exits on its own.
3801        let handle = tokio::spawn(futures::future::pending::<()>());
3802        let svc = make_test_svc(tx, handle, false);
3803
3804        // Keep a second clone alive so Arc::try_unwrap will fail.
3805        let _clone = svc.clone();
3806
3807        // shutdown() on this clone should return immediately (not hang).
3808        svc.shutdown().await;
3809
3810        // _clone still holds the Arc; worker is still running.
3811        // Drop the clone to let the worker task get cleaned up.
3812    }
3813
3814    #[tokio::test]
3815    async fn is_alive_tracks_worker_exit() {
3816        // A running worker reports alive.
3817        let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3818        let alive = make_test_svc(tx, tokio::spawn(futures::future::pending::<()>()), false);
3819        assert!(alive.is_alive());
3820
3821        // A worker that has returned reports not-alive once the task finishes.
3822        let (tx2, _rx2) = mpsc::channel::<WorkerRequest>(1);
3823        let dead = make_test_svc(tx2, tokio::spawn(async {}), false);
3824        // Let the empty worker task complete before checking.
3825        for _ in 0..10 {
3826            if !dead.is_alive() {
3827                break;
3828            }
3829            tokio::task::yield_now().await;
3830        }
3831        assert!(!dead.is_alive());
3832    }
3833
3834    #[tokio::test]
3835    async fn queue_full_returns_queue_full_error() {
3836        // Fill the channel (capacity 1), then verify the next call returns QueueFull.
3837        let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3838        // Fill the one slot without receiving.
3839        let (dummy_tx, _dummy_rx) = oneshot::channel();
3840        tx.try_send(WorkerRequest::Single {
3841            frame: Frame::SimpleString(b"PING"[..].into()),
3842            response_tx: dummy_tx,
3843        })
3844        .unwrap();
3845
3846        let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
3847
3848        let frame = Frame::SimpleString(b"PING"[..].into());
3849        let err = svc.call(frame).await.unwrap_err();
3850        assert!(
3851            matches!(err, RedisError::QueueFull),
3852            "expected QueueFull, got {err:?}"
3853        );
3854    }
3855
3856    #[tokio::test]
3857    async fn backpressure_poll_ready_pends_when_full() {
3858        // Default (back-pressure) mode: a full queue makes poll_ready pend
3859        // rather than returning QueueFull.
3860        let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3861        // Occupy the single slot without draining it.
3862        let (dummy_tx, _dummy_rx) = oneshot::channel();
3863        tx.try_send(WorkerRequest::Single {
3864            frame: Frame::SimpleString(b"PING"[..].into()),
3865            response_tx: dummy_tx,
3866        })
3867        .unwrap();
3868
3869        let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3870        let pending = std::future::poll_fn(|cx| Poll::Ready(svc.poll_ready(cx).is_pending())).await;
3871        assert!(
3872            pending,
3873            "poll_ready must pend (back-pressure) when the queue is full"
3874        );
3875    }
3876
3877    #[tokio::test]
3878    async fn reserved_pipeline_consumes_poll_ready_slot_as_one_request() {
3879        let (tx, mut rx) = mpsc::channel::<WorkerRequest>(1);
3880        let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3881
3882        futures::future::poll_fn(|cx| svc.poll_ready(cx))
3883            .await
3884            .unwrap();
3885
3886        let frames = vec![
3887            Frame::SimpleString(b"CLIENT CACHING YES"[..].into()),
3888            Frame::SimpleString(b"GET key"[..].into()),
3889        ];
3890        let response = svc.call_reserved_pipeline(frames.clone());
3891
3892        let request = rx.recv().await.expect("reserved request should be queued");
3893        match request {
3894            WorkerRequest::Multi {
3895                frames: actual,
3896                response_tx,
3897            } => {
3898                assert_eq!(actual, frames);
3899                response_tx
3900                    .send(Ok(vec![
3901                        Frame::SimpleString(b"OK"[..].into()),
3902                        Frame::BulkString(Some(b"value"[..].into())),
3903                    ]))
3904                    .unwrap();
3905            }
3906            WorkerRequest::Single { .. } => panic!("reserved pipeline was split into a single"),
3907        }
3908
3909        assert_eq!(response.await.unwrap().len(), 2);
3910    }
3911
3912    #[tokio::test]
3913    async fn local_response_can_release_poll_ready_reservation() {
3914        let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
3915        let mut svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
3916
3917        assert_eq!(tx.capacity(), 1);
3918        futures::future::poll_fn(|cx| svc.poll_ready(cx))
3919            .await
3920            .unwrap();
3921        assert_eq!(tx.capacity(), 0, "poll_ready should reserve capacity");
3922
3923        svc.release_reservation();
3924        assert_eq!(
3925            tx.capacity(),
3926            1,
3927            "a locally satisfied request must restore queue capacity"
3928        );
3929    }
3930
3931    #[tokio::test]
3932    async fn backpressure_poll_ready_errors_when_closed() {
3933        // A closed channel surfaces as a (retryable) connection error from
3934        // poll_ready in back-pressure mode -- not a pend.
3935        let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
3936        drop(rx);
3937        let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
3938        let ready = std::future::poll_fn(|cx| svc.poll_ready(cx)).await;
3939        let err = ready.unwrap_err();
3940        assert!(
3941            err.is_retryable(),
3942            "closed-channel error should be retryable"
3943        );
3944    }
3945
3946    #[tokio::test]
3947    async fn queue_full_not_retryable() {
3948        assert!(!RedisError::QueueFull.is_retryable());
3949    }
3950
3951    #[tokio::test]
3952    async fn queue_full_not_connection_error() {
3953        assert!(!RedisError::QueueFull.is_connection_error());
3954    }
3955
3956    #[test]
3957    fn config_queue_capacity_default() {
3958        let config = AutoPipelineConfig::default();
3959        assert_eq!(config.queue_capacity, 1024);
3960    }
3961
3962    #[tokio::test]
3963    async fn queue_depth_zero_when_empty() {
3964        // A fresh channel with nothing sent should report depth 0.
3965        let (tx, _rx) = mpsc::channel::<WorkerRequest>(64);
3966        let svc = make_test_svc(tx, tokio::spawn(async {}), false);
3967        assert_eq!(svc.queue_depth(), 0);
3968    }
3969
3970    #[tokio::test]
3971    async fn queue_depth_increases_with_pending_requests() {
3972        // With no receiver draining, each enqueued request raises the depth.
3973        let (tx, _rx) = mpsc::channel::<WorkerRequest>(10);
3974        let svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
3975
3976        assert_eq!(svc.queue_depth(), 0);
3977
3978        // Manually enqueue a request to simulate a queued, unconsumed item.
3979        let (dummy_tx, _dummy_rx) = oneshot::channel();
3980        tx.try_send(WorkerRequest::Single {
3981            frame: Frame::SimpleString(b"PING"[..].into()),
3982            response_tx: dummy_tx,
3983        })
3984        .unwrap();
3985
3986        assert_eq!(svc.queue_depth(), 1);
3987    }
3988}