Skip to main content

redis_tower/
pool.rs

1//! Tower-native connection pool.
2//!
3//! Pools N connections and dispatches commands via round-robin or random
4//! strategy. Generic over any connection type that implements
5//! [`RedisExecutor`], so it works uniformly with standalone, cluster,
6//! and sentinel connections.
7//!
8//! # Why pool the client, not the node
9//!
10//! For cluster deployments, each pooled entry is a complete
11//! `ClusterConnection` that manages its own node topology and redirect
12//! handling internally. The pool dispatches across N independent cluster
13//! clients. This avoids the common pitfall (seen in redis-rs + bb8) where
14//! individual node connections are pooled separately from cluster routing.
15//!
16//! # Example
17//!
18//! ```no_run
19//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
20//! use redis_tower::pool::ConnectionPool;
21//! use redis_tower::RedisConnection;
22//! use redis_tower::commands::Set;
23//!
24//! // Standalone pool
25//! let pool = ConnectionPool::connect(4, || async {
26//!     RedisConnection::connect("127.0.0.1:6379").await
27//! }).await?;
28//!
29//! // Use from multiple tasks
30//! let p = pool.clone();
31//! tokio::spawn(async move {
32//!     p.execute(Set::new("key", "val")).await.unwrap();
33//! });
34//! # let _ = pool;
35//! # Ok(())
36//! # }
37//! ```
38
39use std::future::Future;
40use std::pin::Pin;
41use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering};
42use std::sync::{Arc, RwLock};
43use std::time::{Duration, Instant};
44
45use redis_tower_commands::{Info, Ping, Role};
46use redis_tower_core::{Command, Frame, RedisConnection, RedisError};
47#[cfg(test)]
48use tokio::sync::MappedMutexGuard;
49use tokio::sync::{Mutex, MutexGuard, watch};
50use tokio::task::JoinHandle;
51use tokio::time::Instant as TokioInstant;
52
53use crate::credentials::{
54    CredentialReauthenticationHandle, Credentials, StreamingCredentialProvider,
55    spawn_credential_reauthentication as spawn_reauthentication_task,
56};
57use crate::executor::RedisExecutor;
58use crate::metrics_layer::MetricsRecorder;
59
60/// A type-erased factory for creating pooled connections.
61///
62/// Implement this trait to give a [`ConnectionPool`] the ability to replace
63/// dead connections after a failed health-check PING. When a PING fails and
64/// a factory is present, the pool calls [`PoolFactory::create`] to obtain a
65/// fresh connection and substitutes it into the dead slot before proceeding.
66pub trait PoolFactory: Send + Sync + 'static {
67    /// The connection type this factory creates.
68    type Connection: RedisExecutor + Send + 'static;
69
70    /// Create a new connection.
71    fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>>;
72}
73
74/// Health state last observed by an active pool prober.
75#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub enum PoolHealthState {
77    /// The slot has not been probed since it was created or replaced.
78    Unknown,
79    /// The most recent probe succeeded and satisfied its policy.
80    Healthy,
81    /// The most recent probe failed or did not satisfy its policy.
82    Unhealthy,
83}
84
85impl PoolHealthState {
86    fn as_u8(self) -> u8 {
87        match self {
88            Self::Unknown => 0,
89            Self::Healthy => 1,
90            Self::Unhealthy => 2,
91        }
92    }
93
94    fn from_u8(value: u8) -> Self {
95        match value {
96            1 => Self::Healthy,
97            2 => Self::Unhealthy,
98            _ => Self::Unknown,
99        }
100    }
101}
102
103/// Redis replication role used by [`RoleHealthProbe`].
104#[derive(Debug, Clone, Copy, PartialEq, Eq)]
105pub enum RedisRole {
106    /// A writable primary (Redis reports this as `master`).
107    Primary,
108    /// A read-only replica (Redis may report `slave` or `replica`).
109    Replica,
110    /// A Sentinel process.
111    Sentinel,
112}
113
114/// Bounded probe kind used by metrics backends.
115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
116pub enum HealthProbeKind {
117    /// `PING` liveness probe.
118    Ping,
119    /// `ROLE` role-verification probe.
120    Role,
121    /// `INFO replication` link and offset-lag probe.
122    ReplicationLag,
123    /// User-supplied probe.
124    Custom,
125}
126
127impl HealthProbeKind {
128    /// Stable low-cardinality label for metrics and logs.
129    pub const fn as_str(self) -> &'static str {
130        match self {
131            Self::Ping => "ping",
132            Self::Role => "role",
133            Self::ReplicationLag => "replication_lag",
134            Self::Custom => "custom",
135        }
136    }
137}
138
139/// Successful output from a [`HealthProbe`].
140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub struct HealthProbeResult {
142    /// Whether the connection satisfies the probe's health policy.
143    pub healthy: bool,
144    /// Replication lag in bytes when the probe observes it.
145    pub replication_lag_bytes: Option<u64>,
146}
147
148impl HealthProbeResult {
149    /// A healthy result without a replication-lag observation.
150    pub const fn healthy() -> Self {
151        Self {
152            healthy: true,
153            replication_lag_bytes: None,
154        }
155    }
156
157    /// An unhealthy result without a replication-lag observation.
158    pub const fn unhealthy() -> Self {
159        Self {
160            healthy: false,
161            replication_lag_bytes: None,
162        }
163    }
164
165    fn with_replication_lag(healthy: bool, replication_lag_bytes: u64) -> Self {
166        Self {
167            healthy,
168            replication_lag_bytes: Some(replication_lag_bytes),
169        }
170    }
171}
172
173/// Asynchronous health check for one pooled connection.
174///
175/// Implementations should perform one bounded logical check and return a
176/// structured observation. The prober applies its own timeout, records the
177/// result, and updates [`PoolStats`].
178pub trait HealthProbe<S>: Send + Sync + 'static {
179    /// Bounded probe kind used by metrics.
180    fn kind(&self) -> HealthProbeKind {
181        HealthProbeKind::Custom
182    }
183
184    /// Check one connection.
185    fn probe<'a>(
186        &'a self,
187        connection: &'a mut S,
188    ) -> Pin<Box<dyn Future<Output = Result<HealthProbeResult, RedisError>> + Send + 'a>>;
189}
190
191/// Default active liveness probe, implemented with Redis `PING`.
192#[derive(Debug, Default, Clone, Copy)]
193pub struct PingHealthProbe;
194
195impl<S> HealthProbe<S> for PingHealthProbe
196where
197    S: RedisExecutor + Send + 'static,
198{
199    fn kind(&self) -> HealthProbeKind {
200        HealthProbeKind::Ping
201    }
202
203    fn probe<'a>(
204        &'a self,
205        connection: &'a mut S,
206    ) -> Pin<Box<dyn Future<Output = Result<HealthProbeResult, RedisError>> + Send + 'a>> {
207        Box::pin(async move {
208            connection.execute(Ping::new()).await?;
209            Ok(HealthProbeResult::healthy())
210        })
211    }
212}
213
214/// Active probe that verifies the role returned by Redis `ROLE`.
215#[derive(Debug, Clone, Copy)]
216pub struct RoleHealthProbe {
217    expected: RedisRole,
218}
219
220impl RoleHealthProbe {
221    /// Require `expected` from every probed connection.
222    pub const fn new(expected: RedisRole) -> Self {
223        Self { expected }
224    }
225}
226
227impl<S> HealthProbe<S> for RoleHealthProbe
228where
229    S: RedisExecutor + Send + 'static,
230{
231    fn kind(&self) -> HealthProbeKind {
232        HealthProbeKind::Role
233    }
234
235    fn probe<'a>(
236        &'a self,
237        connection: &'a mut S,
238    ) -> Pin<Box<dyn Future<Output = Result<HealthProbeResult, RedisError>> + Send + 'a>> {
239        Box::pin(async move {
240            let frame = connection.execute(Role::new()).await?;
241            let observed = parse_role_frame(&frame)?;
242            Ok(if observed == self.expected {
243                HealthProbeResult::healthy()
244            } else {
245                HealthProbeResult::unhealthy()
246            })
247        })
248    }
249}
250
251/// Active `INFO replication` probe with a maximum permitted offset lag.
252///
253/// Run this probe against a primary. Redis reports the primary's current
254/// replication offset and each directly connected replica's offset in the
255/// primary's `INFO replication` response, which lets the probe calculate the
256/// largest byte lag without contacting a second server. A primary with no
257/// connected replicas is unhealthy. Replica-local INFO does not contain the
258/// upstream primary's current offset, so probing a replica is also unhealthy
259/// rather than reporting a misleading zero-byte lag.
260#[derive(Debug, Clone, Copy)]
261pub struct ReplicationLagHealthProbe {
262    max_lag_bytes: u64,
263}
264
265impl ReplicationLagHealthProbe {
266    /// Mark replicas unhealthy when their replication offset trails by more
267    /// than `max_lag_bytes`, or while their primary link is down/synchronizing.
268    pub const fn new(max_lag_bytes: u64) -> Self {
269        Self { max_lag_bytes }
270    }
271}
272
273impl<S> HealthProbe<S> for ReplicationLagHealthProbe
274where
275    S: RedisExecutor + Send + 'static,
276{
277    fn kind(&self) -> HealthProbeKind {
278        HealthProbeKind::ReplicationLag
279    }
280
281    fn probe<'a>(
282        &'a self,
283        connection: &'a mut S,
284    ) -> Pin<Box<dyn Future<Output = Result<HealthProbeResult, RedisError>> + Send + 'a>> {
285        Box::pin(async move {
286            let info = connection
287                .execute(Info::new().section("replication"))
288                .await?;
289            let observation = parse_replication_info(&info)?;
290            let healthy = observation.link_up
291                && !observation.sync_in_progress
292                && observation.lag_bytes <= self.max_lag_bytes;
293            Ok(HealthProbeResult::with_replication_lag(
294                healthy,
295                observation.lag_bytes,
296            ))
297        })
298    }
299}
300
301/// Configuration for an explicitly spawned active health prober.
302#[derive(Debug, Clone, Copy)]
303pub struct HealthProberConfig {
304    /// Delay between complete sweeps of the pool.
305    pub interval: Duration,
306    /// Maximum duration of one connection probe.
307    pub timeout: Duration,
308}
309
310impl Default for HealthProberConfig {
311    fn default() -> Self {
312        Self {
313            interval: Duration::from_secs(10),
314            timeout: Duration::from_secs(2),
315        }
316    }
317}
318
319impl HealthProberConfig {
320    /// Set the delay between complete pool sweeps.
321    pub fn interval(mut self, interval: Duration) -> Self {
322        self.interval = interval;
323        self
324    }
325
326    /// Set the timeout applied to each connection probe.
327    pub fn timeout(mut self, timeout: Duration) -> Self {
328        self.timeout = timeout;
329        self
330    }
331}
332
333/// A snapshot of current [`ConnectionPool`] utilization.
334///
335/// Obtained via [`ConnectionPool::stats`]. All values are point-in-time
336/// reads of the underlying atomic counters; concurrent commands may cause
337/// values to change between calls.
338#[derive(Debug, Clone)]
339pub struct PoolStats {
340    /// Total number of connections in the pool.
341    pub size: usize,
342    /// Number of connections with zero in-flight commands (idle).
343    pub idle_count: usize,
344    /// Total in-flight command count across all connections.
345    pub total_inflight: usize,
346    /// Highest in-flight count on a single connection.
347    pub max_inflight: usize,
348    /// Configured lower bound for idle reaping.
349    pub min_size: usize,
350    /// Configured upper bound for contention-driven growth.
351    pub max_size: usize,
352    /// Slots whose most recent active probe was healthy.
353    pub healthy_count: usize,
354    /// Slots whose most recent active probe was unhealthy.
355    pub unhealthy_count: usize,
356    /// Slots that have not been actively probed since creation/replacement.
357    pub unknown_health_count: usize,
358    /// Largest replication lag reported by the latest slot observations.
359    pub max_replication_lag_bytes: Option<u64>,
360    /// Cumulative number of idle connections removed by a reaper.
361    pub reaped_connections: u64,
362}
363
364/// Dispatch strategy for distributing commands across pooled connections.
365#[derive(Debug, Clone, Copy, Default)]
366pub enum DispatchStrategy {
367    /// Cycle through connections sequentially (default).
368    #[default]
369    RoundRobin,
370    /// Pick a random connection for each command.
371    Random,
372    /// Pick the connection with the fewest in-flight commands.
373    /// Best for workloads with variable command latency (e.g., mix of
374    /// GET and SORT). Falls back to round-robin on ties.
375    LeastConnections,
376}
377
378/// Configuration for a connection pool.
379#[derive(Clone)]
380pub struct PoolConfig {
381    /// Stable name used to identify this pool in metrics.
382    ///
383    /// Applications with multiple pools should give each one a distinct name
384    /// so their utilization and acquisition metrics remain distinguishable.
385    /// Defaults to `"redis-tower"`.
386    pub name: String,
387    /// Legacy fixed size and fallback for omitted dynamic bounds.
388    ///
389    /// With neither `min_size` nor `max_size`, the pool remains fixed at this
390    /// size. Prefer [`Self::bounds`] when opting into dynamic sizing.
391    pub size: usize,
392    /// Optional minimum number of live connections.
393    ///
394    /// When omitted, [`Self::size`] is used. Idle reaping never shrinks below
395    /// this value. Set this below `max_size` to opt into dynamic sizing.
396    pub min_size: Option<usize>,
397    /// Optional maximum number of live connections.
398    ///
399    /// When omitted, [`Self::size`] is used. A factory-backed pool creates new
400    /// connections up to this bound when every live slot is contended.
401    pub max_size: Option<usize>,
402    /// How long a completely idle connection may remain before an explicitly
403    /// spawned idle reaper removes it.
404    ///
405    /// Setting this value does not spawn a task. Call
406    /// [`ConnectionPool::spawn_idle_reaper`] and retain its owned handle.
407    pub idle_timeout: Option<Duration>,
408    /// How to select which connection handles each command.
409    pub dispatch: DispatchStrategy,
410    /// If set, connections idle longer than this duration are PINGed before use.
411    ///
412    /// This provides lazy health checking: when a connection has been idle
413    /// beyond this interval, a PING is sent before dispatching the actual
414    /// command. If the PING fails and the pool has a factory, the dead slot
415    /// is replaced before the command is retried. If no factory is present,
416    /// the error is returned to the caller.
417    pub health_check_interval: Option<Duration>,
418    /// Maximum time to wait for a connection slot to become available.
419    ///
420    /// When all connections in the pool are busy, new callers block waiting
421    /// for a slot to free up. If the wait exceeds this duration,
422    /// [`RedisError::PoolAcquisitionTimeout`] is returned, so pool exhaustion
423    /// fails fast as a timeout instead of stalling the caller indefinitely.
424    ///
425    /// Defaults to [`PoolConfig::DEFAULT_ACQUISITION_TIMEOUT`] (5 seconds).
426    /// Set it to `None` -- via [`PoolConfig::disable_acquisition_timeout`] --
427    /// to wait forever, restoring the previous unbounded behavior.
428    /// [`Duration::ZERO`] is distinct from `None`: it permits an immediately
429    /// available slot but never waits for a busy one.
430    /// A command wrapped in [`redis_tower_core::WithDeadline`] can impose an
431    /// earlier limit; the earliest deadline wins.
432    pub acquisition_timeout: Option<Duration>,
433    /// Optional recorder for pool lifecycle metrics.
434    ///
435    /// When set, the pool reports connection acquisition latency and timeouts,
436    /// lazy health-check failures, and successful connection replacements.
437    pub metrics_recorder: Option<Arc<dyn MetricsRecorder>>,
438}
439
440impl std::fmt::Debug for PoolConfig {
441    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
442        f.debug_struct("PoolConfig")
443            .field("name", &self.name)
444            .field("size", &self.size)
445            .field("min_size", &self.min_size)
446            .field("max_size", &self.max_size)
447            .field("idle_timeout", &self.idle_timeout)
448            .field("dispatch", &self.dispatch)
449            .field("health_check_interval", &self.health_check_interval)
450            .field("acquisition_timeout", &self.acquisition_timeout)
451            .field(
452                "metrics_recorder",
453                &self.metrics_recorder.as_ref().map(|_| "<recorder>"),
454            )
455            .finish()
456    }
457}
458
459impl Default for PoolConfig {
460    fn default() -> Self {
461        Self {
462            name: "redis-tower".to_owned(),
463            size: 4,
464            min_size: None,
465            max_size: None,
466            idle_timeout: None,
467            dispatch: DispatchStrategy::RoundRobin,
468            health_check_interval: None,
469            acquisition_timeout: Some(Self::DEFAULT_ACQUISITION_TIMEOUT),
470            metrics_recorder: None,
471        }
472    }
473}
474
475impl PoolConfig {
476    /// Default upper bound on how long a caller waits for a connection slot.
477    ///
478    /// Used by [`PoolConfig::default`]. Five seconds is long enough to ride
479    /// out brief contention but short enough that genuine pool exhaustion
480    /// surfaces as a [`RedisError::PoolAcquisitionTimeout`] rather than an
481    /// unbounded stall that masquerades as a hang.
482    pub const DEFAULT_ACQUISITION_TIMEOUT: Duration = Duration::from_secs(5);
483
484    /// Set the stable name used to identify this pool in metrics.
485    pub fn name(mut self, name: impl Into<String>) -> Self {
486        self.name = name.into();
487        self
488    }
489
490    /// Set the pool size.
491    pub fn size(mut self, size: usize) -> Self {
492        self.size = size;
493        self
494    }
495
496    /// Set the minimum number of live connections maintained by the pool.
497    pub fn min_size(mut self, min_size: usize) -> Self {
498        self.min_size = Some(min_size);
499        self
500    }
501
502    /// Set the maximum number of live connections permitted in the pool.
503    pub fn max_size(mut self, max_size: usize) -> Self {
504        self.max_size = Some(max_size);
505        self
506    }
507
508    /// Set both dynamic sizing bounds.
509    pub fn bounds(mut self, min_size: usize, max_size: usize) -> Self {
510        self.min_size = Some(min_size);
511        self.max_size = Some(max_size);
512        self
513    }
514
515    /// Set the idle duration after which a reaper may remove a connection.
516    ///
517    /// This only configures policy. No background task exists until
518    /// [`ConnectionPool::spawn_idle_reaper`] is called.
519    pub fn idle_timeout(mut self, idle_timeout: Duration) -> Self {
520        self.idle_timeout = Some(idle_timeout);
521        self
522    }
523
524    /// Set the dispatch strategy.
525    pub fn dispatch(mut self, strategy: DispatchStrategy) -> Self {
526        self.dispatch = strategy;
527        self
528    }
529
530    /// Set the health check interval.
531    ///
532    /// If set, connections idle longer than this are PINGed before use
533    /// to verify they are still alive.
534    pub fn health_check_interval(mut self, interval: Duration) -> Self {
535        self.health_check_interval = Some(interval);
536        self
537    }
538
539    /// Set the maximum time to wait for a connection slot.
540    ///
541    /// If all connections are busy when a command is submitted and a slot
542    /// does not free up within `timeout`, the call returns
543    /// [`RedisError::PoolAcquisitionTimeout`]. Overrides the bounded
544    /// [`PoolConfig::DEFAULT_ACQUISITION_TIMEOUT`]. Pass [`Duration::ZERO`] for
545    /// fail-fast acquisition that never waits; use
546    /// [`Self::disable_acquisition_timeout`] to wait without a bound.
547    pub fn acquisition_timeout(mut self, timeout: Duration) -> Self {
548        self.acquisition_timeout = Some(timeout);
549        self
550    }
551
552    /// Wait forever for a connection slot, disabling the acquisition timeout.
553    ///
554    /// This restores the previous behavior where a saturated pool blocks the
555    /// caller indefinitely. Prefer the bounded
556    /// [`default`](PoolConfig::default) unless an unbounded wait is genuinely
557    /// what you want -- an unbounded wait turns pool exhaustion into a silent
558    /// hang rather than a surfaced [`RedisError::PoolAcquisitionTimeout`].
559    pub fn disable_acquisition_timeout(mut self) -> Self {
560        self.acquisition_timeout = None;
561        self
562    }
563
564    /// Set the recorder that receives pool lifecycle metric events.
565    pub fn metrics_recorder(mut self, recorder: Arc<dyn MetricsRecorder>) -> Self {
566        self.metrics_recorder = Some(recorder);
567        self
568    }
569}
570
571fn frame_text(frame: &Frame) -> Option<&[u8]> {
572    match frame {
573        Frame::SimpleString(value)
574        | Frame::BulkString(Some(value))
575        | Frame::VerbatimString(_, value) => Some(value.as_ref()),
576        _ => None,
577    }
578}
579
580fn parse_role_frame(frame: &Frame) -> Result<RedisRole, RedisError> {
581    let Frame::Array(Some(parts)) = frame else {
582        return Err(RedisError::UnexpectedResponse {
583            expected: "ROLE array",
584            actual: format!("{frame:?}"),
585        });
586    };
587    let Some(role) = parts.first().and_then(frame_text) else {
588        return Err(RedisError::UnexpectedResponse {
589            expected: "ROLE array beginning with a role name",
590            actual: format!("{frame:?}"),
591        });
592    };
593    match role {
594        b"master" => Ok(RedisRole::Primary),
595        b"slave" | b"replica" => Ok(RedisRole::Replica),
596        b"sentinel" => Ok(RedisRole::Sentinel),
597        _ => Err(RedisError::UnexpectedResponse {
598            expected: "master, slave/replica, or sentinel ROLE",
599            actual: String::from_utf8_lossy(role).into_owned(),
600        }),
601    }
602}
603
604struct ReplicationObservation {
605    link_up: bool,
606    sync_in_progress: bool,
607    lag_bytes: u64,
608}
609
610fn parse_replication_info(info: &str) -> Result<ReplicationObservation, RedisError> {
611    let field = |name: &str| {
612        info.lines().find_map(|line| {
613            let (key, value) = line.trim_end_matches('\r').split_once(':')?;
614            (key == name).then_some(value)
615        })
616    };
617
618    if field("role") != Some("master") {
619        return Err(RedisError::UnexpectedResponse {
620            expected: "primary INFO replication with connected replica offsets",
621            actual: info.to_owned(),
622        });
623    }
624
625    let primary_offset = field("master_repl_offset")
626        .and_then(|value| value.parse::<u64>().ok())
627        .ok_or_else(|| RedisError::UnexpectedResponse {
628            expected: "numeric master_repl_offset in INFO replication",
629            actual: info.to_owned(),
630        })?;
631    let connected_replicas = field("connected_slaves")
632        .or_else(|| field("connected_replicas"))
633        .and_then(|value| value.parse::<usize>().ok())
634        .ok_or_else(|| RedisError::UnexpectedResponse {
635            expected: "numeric connected_slaves in primary INFO replication",
636            actual: info.to_owned(),
637        })?;
638
639    let mut replica_offsets = Vec::with_capacity(connected_replicas);
640    let mut all_online = true;
641    for line in info.lines() {
642        let Some((key, value)) = line.trim_end_matches('\r').split_once(':') else {
643            continue;
644        };
645        let suffix = key
646            .strip_prefix("slave")
647            .or_else(|| key.strip_prefix("replica"));
648        if !suffix.is_some_and(|suffix| {
649            !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit())
650        }) {
651            continue;
652        }
653
654        let replica_field = |name: &str| {
655            value.split(',').find_map(|entry| {
656                let (key, value) = entry.split_once('=')?;
657                (key == name).then_some(value)
658            })
659        };
660        let state = replica_field("state").ok_or_else(|| RedisError::UnexpectedResponse {
661            expected: "state in primary replica INFO entry",
662            actual: line.to_owned(),
663        })?;
664        let offset = replica_field("offset")
665            .and_then(|value| value.parse::<u64>().ok())
666            .ok_or_else(|| RedisError::UnexpectedResponse {
667                expected: "numeric offset in primary replica INFO entry",
668                actual: line.to_owned(),
669            })?;
670        all_online &= state == "online";
671        replica_offsets.push(offset);
672    }
673
674    if replica_offsets.len() != connected_replicas {
675        return Err(RedisError::UnexpectedResponse {
676            expected: "one parseable INFO entry per connected replica",
677            actual: info.to_owned(),
678        });
679    }
680
681    Ok(ReplicationObservation {
682        link_up: connected_replicas > 0 && all_online,
683        sync_in_progress: !replica_offsets.is_empty() && !all_online,
684        lag_bytes: replica_offsets
685            .into_iter()
686            .map(|offset| primary_offset.saturating_sub(offset))
687            .max()
688            .unwrap_or(0),
689    })
690}
691
692/// Return the current epoch time in milliseconds.
693fn now_millis() -> u64 {
694    std::time::SystemTime::now()
695        .duration_since(std::time::UNIX_EPOCH)
696        .unwrap_or_default()
697        .as_millis() as u64
698}
699
700/// The high bit of `admission_state` permanently marks the pool closed; the
701/// remaining bits count accepted operations. Combining both values in one
702/// atomic makes admission linearizable with [`ConnectionPool::close`].
703const POOL_CLOSED_BIT: usize = 1usize << (usize::BITS - 1);
704const POOL_ACTIVE_MASK: usize = !POOL_CLOSED_BIT;
705
706/// Shared state behind the pool's Arc.
707struct PoolSlot<S> {
708    connection: Mutex<Option<S>>,
709    active: AtomicBool,
710    inflight: AtomicUsize,
711    last_used: AtomicU64,
712    health: AtomicU8,
713    replication_lag_bytes: AtomicU64,
714}
715
716impl<S> PoolSlot<S> {
717    fn active(connection: S, now: u64) -> Self {
718        Self {
719            connection: Mutex::new(Some(connection)),
720            active: AtomicBool::new(true),
721            inflight: AtomicUsize::new(0),
722            last_used: AtomicU64::new(now),
723            health: AtomicU8::new(PoolHealthState::Unknown.as_u8()),
724            replication_lag_bytes: AtomicU64::new(u64::MAX),
725        }
726    }
727
728    fn vacant(now: u64) -> Self {
729        Self {
730            connection: Mutex::new(None),
731            active: AtomicBool::new(false),
732            inflight: AtomicUsize::new(0),
733            last_used: AtomicU64::new(now),
734            health: AtomicU8::new(PoolHealthState::Unknown.as_u8()),
735            replication_lag_bytes: AtomicU64::new(u64::MAX),
736        }
737    }
738
739    #[cfg(test)]
740    async fn lock(&self) -> MappedMutexGuard<'_, S> {
741        MutexGuard::map(self.connection.lock().await, |connection| {
742            connection
743                .as_mut()
744                .expect("active pool slot must contain a connection")
745        })
746    }
747}
748
749struct PoolInner<S> {
750    /// Stable pool name attached to metric events.
751    name: String,
752    /// Fixed-capacity slot table. `active` determines which slots participate
753    /// in dispatch; inactive slots contain `None` and can be filled on growth.
754    connections: Vec<PoolSlot<S>>,
755    min_size: usize,
756    max_size: usize,
757    idle_timeout_ms: Option<u64>,
758    reaped_connections: AtomicU64,
759    /// Serializes slot creation and reaping. Connection creation may await,
760    /// so this is an async mutex separate from the short topology lock.
761    scale_lock: Mutex<()>,
762    /// Protects active/inactive transitions against reservation selection.
763    topology_lock: RwLock<()>,
764    index: AtomicUsize,
765    dispatch: DispatchStrategy,
766    /// Health check interval in milliseconds, or 0 if disabled.
767    health_check_interval_ms: u64,
768    /// Acquisition timeout. `None` means unlimited; zero means do not wait.
769    acquisition_timeout: Option<Duration>,
770    /// Optional factory used to replace dead connections after a failed PING.
771    factory: Option<Arc<dyn ErasedPoolFactory<Connection = S>>>,
772    /// Optional recorder for pool lifecycle events.
773    metrics_recorder: Option<Arc<dyn MetricsRecorder>>,
774    /// Closed flag and pool-wide accepted-operation count in one atomic word.
775    /// The high bit is the closed flag; the remaining bits are the count.
776    admission_state: AtomicUsize,
777}
778
779/// Owns one pool-wide admission until an accepted operation finishes.
780struct PoolAdmission<'a> {
781    state: &'a AtomicUsize,
782}
783
784impl<'a> PoolAdmission<'a> {
785    fn try_new(state: &'a AtomicUsize) -> Result<Self, RedisError> {
786        let mut current = state.load(Ordering::Acquire);
787        loop {
788            if current & POOL_CLOSED_BIT != 0 {
789                return Err(RedisError::ConnectionClosed);
790            }
791            assert!(
792                current & POOL_ACTIVE_MASK != POOL_ACTIVE_MASK,
793                "pool active-operation counter overflow"
794            );
795
796            match state.compare_exchange_weak(
797                current,
798                current + 1,
799                Ordering::AcqRel,
800                Ordering::Acquire,
801            ) {
802                Ok(_) => return Ok(Self { state }),
803                Err(observed) => current = observed,
804            }
805        }
806    }
807}
808
809impl Drop for PoolAdmission<'_> {
810    fn drop(&mut self) {
811        let previous = self.state.fetch_sub(1, Ordering::AcqRel);
812        debug_assert!(previous & POOL_ACTIVE_MASK > 0);
813    }
814}
815
816#[derive(Clone, Copy)]
817struct AcquisitionBudget {
818    started: Instant,
819    started_at: TokioInstant,
820    request_deadline: Option<TokioInstant>,
821    pool_timeout: Option<Duration>,
822}
823
824impl AcquisitionBudget {
825    fn new(pool_timeout: Option<Duration>, request_deadline: Option<TokioInstant>) -> Self {
826        Self {
827            started: Instant::now(),
828            started_at: TokioInstant::now(),
829            request_deadline,
830            pool_timeout,
831        }
832    }
833
834    fn request_expired(self) -> bool {
835        self.request_deadline
836            .is_some_and(|deadline| deadline <= TokioInstant::now())
837    }
838
839    fn effective_wait_deadline(self) -> Option<(TokioInstant, Option<Duration>)> {
840        let pool_deadline = self.pool_timeout.map(|duration| self.started_at + duration);
841        match (self.request_deadline, pool_deadline) {
842            (Some(request), Some(pool)) if request <= pool => Some((request, None)),
843            (Some(_request), Some(pool)) => Some((pool, self.pool_timeout)),
844            (Some(request), None) => Some((request, None)),
845            (None, Some(pool)) => Some((pool, self.pool_timeout)),
846            (None, None) => None,
847        }
848    }
849
850    fn timeout_error(self, pool_size: usize, pool_timeout: Option<Duration>) -> RedisError {
851        match pool_timeout {
852            Some(waited) => RedisError::PoolAcquisitionTimeout { waited, pool_size },
853            None => RedisError::CommandTimeout,
854        }
855    }
856
857    async fn wait<F, T>(self, future: F, pool_size: usize) -> Result<T, RedisError>
858    where
859        F: Future<Output = T>,
860    {
861        let Some((deadline, pool_timeout)) = self.effective_wait_deadline() else {
862            return Ok(future.await);
863        };
864        if deadline <= TokioInstant::now() {
865            return Err(self.timeout_error(pool_size, pool_timeout));
866        }
867        match tokio::time::timeout_at(deadline, future).await {
868            Ok(value) if deadline > TokioInstant::now() => Ok(value),
869            Ok(_) | Err(_) => Err(self.timeout_error(pool_size, pool_timeout)),
870        }
871    }
872
873    fn elapsed(self) -> Duration {
874        self.started.elapsed()
875    }
876}
877
878/// Owns one in-flight reservation until a command finishes or is cancelled.
879///
880/// Keeping the decrement in `Drop` makes every await in the execution path
881/// cancellation-safe: dropping the command future releases its reservation
882/// whether it is waiting for a connection, checking health, replacing a dead
883/// connection, or awaiting the command response.
884struct InflightReservation<'a, S> {
885    slot: &'a PoolSlot<S>,
886    index: usize,
887    _admission: PoolAdmission<'a>,
888}
889
890impl<'a, S> InflightReservation<'a, S> {
891    fn new(slot: &'a PoolSlot<S>, index: usize, admission: PoolAdmission<'a>) -> Self {
892        slot.inflight.fetch_add(1, Ordering::Release);
893        Self {
894            slot,
895            index,
896            _admission: admission,
897        }
898    }
899
900    fn index(&self) -> usize {
901        self.index
902    }
903
904    /// Move this reservation to an alternate slot selected by `acquire`.
905    fn transfer_to(&mut self, slot: &'a PoolSlot<S>, index: usize) {
906        if index == self.index {
907            return;
908        }
909
910        slot.inflight.fetch_add(1, Ordering::Release);
911        self.slot.inflight.fetch_sub(1, Ordering::Release);
912        self.slot = slot;
913        self.index = index;
914    }
915}
916
917impl<S> Drop for InflightReservation<'_, S> {
918    fn drop(&mut self) {
919        let previous = self.slot.inflight.fetch_sub(1, Ordering::Release);
920        debug_assert!(previous > 0);
921    }
922}
923
924/// Object-safe wrapper around [`PoolFactory`] that erases the concrete type.
925///
926/// `PoolFactory` cannot itself be made into a trait object because of the
927/// associated type, so we use this helper to expose the same surface via
928/// `dyn`.
929trait ErasedPoolFactory: Send + Sync + 'static {
930    type Connection: Send + 'static;
931    fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>>;
932}
933
934impl<F: PoolFactory> ErasedPoolFactory for F {
935    type Connection = F::Connection;
936    fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
937        PoolFactory::create(self)
938    }
939}
940
941impl<S> PoolInner<S>
942where
943    S: Send + 'static,
944{
945    fn topology_read(&self) -> std::sync::RwLockReadGuard<'_, ()> {
946        self.topology_lock
947            .read()
948            .unwrap_or_else(std::sync::PoisonError::into_inner)
949    }
950
951    fn topology_write(&self) -> std::sync::RwLockWriteGuard<'_, ()> {
952        self.topology_lock
953            .write()
954            .unwrap_or_else(std::sync::PoisonError::into_inner)
955    }
956
957    fn current_size(&self) -> usize {
958        self.connections
959            .iter()
960            .filter(|slot| slot.active.load(Ordering::Acquire))
961            .count()
962    }
963
964    /// Move an existing reservation only when `index` is still active.
965    ///
966    /// Holding the topology read lock through the in-flight increment makes
967    /// the transfer atomic with idle reaping: a reaper either removes the slot
968    /// first and this returns `false`, or observes the new reservation and
969    /// leaves the slot active.
970    fn transfer_reservation_if_active<'a>(
971        &'a self,
972        reservation: &mut InflightReservation<'a, S>,
973        index: usize,
974    ) -> bool {
975        let topology = self.topology_read();
976        if !self.connections[index].active.load(Ordering::Acquire) {
977            return false;
978        }
979        reservation.transfer_to(&self.connections[index], index);
980        drop(topology);
981        true
982    }
983
984    /// Create and activate one connection when capacity remains.
985    async fn grow_one(&self) -> Result<Option<usize>, RedisError> {
986        let _scale = self.scale_lock.lock().await;
987        if self.current_size() >= self.max_size {
988            return Ok(None);
989        }
990        let Some(factory) = &self.factory else {
991            return Ok(None);
992        };
993        let Some(index) = self
994            .connections
995            .iter()
996            .position(|slot| !slot.active.load(Ordering::Acquire))
997        else {
998            return Ok(None);
999        };
1000
1001        let connection = ErasedPoolFactory::create(factory.as_ref()).await?;
1002        let mut slot = self.connections[index].connection.lock().await;
1003        let _topology = self.topology_write();
1004        if self.connections[index].active.load(Ordering::Acquire) {
1005            return Ok(None);
1006        }
1007        *slot = Some(connection);
1008        self.connections[index]
1009            .last_used
1010            .store(now_millis(), Ordering::Release);
1011        self.connections[index]
1012            .health
1013            .store(PoolHealthState::Unknown.as_u8(), Ordering::Release);
1014        self.connections[index]
1015            .replication_lag_bytes
1016            .store(u64::MAX, Ordering::Release);
1017        self.connections[index]
1018            .active
1019            .store(true, Ordering::Release);
1020        Ok(Some(index))
1021    }
1022
1023    /// Select a live connection according to the dispatch strategy.
1024    async fn reserve_next(
1025        &self,
1026        budget: AcquisitionBudget,
1027    ) -> Result<InflightReservation<'_, S>, RedisError> {
1028        let admission = PoolAdmission::try_new(&self.admission_state)?;
1029        loop {
1030            let topology = self.topology_read();
1031            let active: Vec<usize> = self
1032                .connections
1033                .iter()
1034                .enumerate()
1035                .filter_map(|(index, slot)| slot.active.load(Ordering::Acquire).then_some(index))
1036                .collect();
1037            if !active.is_empty() {
1038                let index = match self.dispatch {
1039                    DispatchStrategy::RoundRobin => {
1040                        active[self.index.fetch_add(1, Ordering::Relaxed) % active.len()]
1041                    }
1042                    DispatchStrategy::Random => {
1043                        // Simple xorshift-based pseudo-random from the atomic
1044                        // counter. It is distribution, not cryptography.
1045                        let mut value = self.index.fetch_add(7, Ordering::Relaxed);
1046                        value ^= value << 13;
1047                        value ^= value >> 7;
1048                        value ^= value << 17;
1049                        active[value % active.len()]
1050                    }
1051                    DispatchStrategy::LeastConnections => *active
1052                        .iter()
1053                        .min_by_key(|index| {
1054                            self.connections[**index].inflight.load(Ordering::Acquire)
1055                        })
1056                        .expect("active slots are non-empty"),
1057                };
1058                let reservation =
1059                    InflightReservation::new(&self.connections[index], index, admission);
1060                drop(topology);
1061                return Ok(reservation);
1062            }
1063            drop(topology);
1064
1065            let grown = match budget.wait(self.grow_one(), self.current_size()).await {
1066                Ok(result) => result,
1067                Err(error) => {
1068                    self.record_acquisition(budget.elapsed(), true);
1069                    return Err(error);
1070                }
1071            };
1072            match grown {
1073                Ok(Some(_)) => {}
1074                Ok(None) => {
1075                    self.record_acquisition(budget.elapsed(), false);
1076                    return Err(RedisError::ConnectionClosed);
1077                }
1078                Err(error) => {
1079                    self.record_acquisition(budget.elapsed(), false);
1080                    return Err(error);
1081                }
1082            }
1083        }
1084    }
1085
1086    /// Acquire a connection guard, avoiding head-of-line blocking.
1087    ///
1088    /// `reservation` owns the strategy-selected slot's in-flight count. If that
1089    /// slot is immediately lockable it is used directly. Otherwise a `try_lock`
1090    /// scan looks for any idle connection so the request is not forced to queue
1091    /// behind a long-running command on the preferred slot while another
1092    /// connection sits free; when a free slot is found the reservation is moved
1093    /// to it. If every slot is busy, the call awaits the preferred slot,
1094    /// honoring the acquisition timeout.
1095    ///
1096    /// Returns the slot index actually acquired together with its guard.
1097    async fn acquire<'a>(
1098        &'a self,
1099        reservation: &mut InflightReservation<'a, S>,
1100        budget: AcquisitionBudget,
1101    ) -> Result<(usize, MutexGuard<'a, Option<S>>), RedisError> {
1102        let preferred = reservation.index();
1103
1104        if budget.request_expired() {
1105            self.record_acquisition(budget.elapsed(), true);
1106            return Err(RedisError::CommandTimeout);
1107        }
1108
1109        // Fast path: the preferred slot is free.
1110        if let Ok(guard) = self.connections[preferred].connection.try_lock() {
1111            if budget.request_expired() {
1112                drop(guard);
1113                self.record_acquisition(budget.elapsed(), true);
1114                return Err(RedisError::CommandTimeout);
1115            }
1116            self.record_acquisition(budget.elapsed(), false);
1117            return Ok((preferred, guard));
1118        }
1119
1120        // Head-of-line-blocking avoidance: the preferred slot is busy, so scan
1121        // the remaining slots for any immediately free connection before
1122        // committing to an await on the busy slot.
1123        let capacity = self.connections.len();
1124        for offset in 1..capacity {
1125            let i = (preferred + offset) % capacity;
1126            if !self.connections[i].active.load(Ordering::Acquire) {
1127                continue;
1128            }
1129            if let Ok(guard) = self.connections[i].connection.try_lock() {
1130                if budget.request_expired() {
1131                    drop(guard);
1132                    self.record_acquisition(budget.elapsed(), true);
1133                    return Err(RedisError::CommandTimeout);
1134                }
1135                let topology = self.topology_read();
1136                if !self.connections[i].active.load(Ordering::Acquire) {
1137                    drop(topology);
1138                    drop(guard);
1139                    continue;
1140                }
1141                reservation.transfer_to(&self.connections[i], i);
1142                drop(topology);
1143                self.record_acquisition(budget.elapsed(), false);
1144                return Ok((i, guard));
1145            }
1146        }
1147
1148        // All live slots are busy. A dynamic factory-backed pool grows on
1149        // observed contention before it queues behind an existing slot.
1150        if self.current_size() < self.max_size {
1151            let growth = budget.wait(self.grow_one(), self.current_size()).await;
1152            match growth {
1153                Ok(Ok(Some(index))) => {
1154                    if self.transfer_reservation_if_active(reservation, index) {
1155                        let guard = match budget
1156                            .wait(
1157                                self.connections[index].connection.lock(),
1158                                self.current_size(),
1159                            )
1160                            .await
1161                        {
1162                            Ok(guard) => guard,
1163                            Err(error) => {
1164                                self.record_acquisition(budget.elapsed(), true);
1165                                return Err(error);
1166                            }
1167                        };
1168                        self.record_acquisition(budget.elapsed(), false);
1169                        return Ok((index, guard));
1170                    }
1171                    // An idle reaper removed the newly activated slot before
1172                    // this caller could reserve it. Fall back to a still-live
1173                    // slot instead of returning an inactive connection.
1174                }
1175                Ok(Ok(None)) | Ok(Err(_)) => {
1176                    // Another caller filled capacity, or opportunistic growth
1177                    // failed. Existing live slots can still serve this request.
1178                }
1179                Err(error) => {
1180                    self.record_acquisition(budget.elapsed(), true);
1181                    return Err(error);
1182                }
1183            }
1184        }
1185
1186        // Every slot is busy. Await the preferred slot until the earliest of
1187        // the command's absolute deadline and the pool's static acquisition
1188        // timeout. Preserve the error that identifies which budget expired.
1189        let guard = match budget
1190            .wait(
1191                self.connections[preferred].connection.lock(),
1192                self.current_size(),
1193            )
1194            .await
1195        {
1196            Ok(guard) => guard,
1197            Err(error) => {
1198                self.record_acquisition(budget.elapsed(), true);
1199                return Err(error);
1200            }
1201        };
1202        self.record_acquisition(budget.elapsed(), false);
1203        Ok((preferred, guard))
1204    }
1205
1206    fn record_acquisition(&self, duration: Duration, timed_out: bool) {
1207        if let Some(recorder) = &self.metrics_recorder {
1208            recorder.pool_acquisition_completed(&self.name, duration, timed_out);
1209        }
1210    }
1211
1212    fn record_health_check_failed(&self) {
1213        if let Some(recorder) = &self.metrics_recorder {
1214            recorder.pool_health_check_failed(&self.name);
1215        }
1216    }
1217
1218    fn record_connection_replaced(&self) {
1219        if let Some(recorder) = &self.metrics_recorder {
1220            recorder.pool_connection_replaced(&self.name);
1221        }
1222    }
1223
1224    fn record_health_probe(
1225        &self,
1226        kind: HealthProbeKind,
1227        duration: Duration,
1228        healthy: bool,
1229        replication_lag_bytes: Option<u64>,
1230    ) {
1231        if let Some(recorder) = &self.metrics_recorder {
1232            recorder.pool_health_probe_completed(
1233                &self.name,
1234                kind,
1235                duration,
1236                healthy,
1237                replication_lag_bytes,
1238            );
1239        }
1240    }
1241
1242    fn record_connections_reaped(&self, count: usize) {
1243        if let Some(recorder) = &self.metrics_recorder {
1244            recorder.pool_connections_reaped(&self.name, count);
1245        }
1246    }
1247
1248    async fn reap_idle_once(&self) -> usize {
1249        let Some(idle_timeout_ms) = self.idle_timeout_ms else {
1250            return 0;
1251        };
1252        let _scale = self.scale_lock.lock().await;
1253        let now = now_millis();
1254        let mut removed = Vec::new();
1255        let topology = self.topology_write();
1256        let mut active = self.current_size();
1257        for slot in self.connections.iter().rev() {
1258            if active <= self.min_size {
1259                break;
1260            }
1261            if !slot.active.load(Ordering::Acquire)
1262                || slot.inflight.load(Ordering::Acquire) != 0
1263                || now.saturating_sub(slot.last_used.load(Ordering::Acquire)) < idle_timeout_ms
1264            {
1265                continue;
1266            }
1267            let Ok(mut connection) = slot.connection.try_lock() else {
1268                continue;
1269            };
1270            if slot.inflight.load(Ordering::Acquire) != 0 {
1271                continue;
1272            }
1273            slot.active.store(false, Ordering::Release);
1274            if let Some(connection) = connection.take() {
1275                removed.push(connection);
1276                active -= 1;
1277            }
1278            slot.health
1279                .store(PoolHealthState::Unknown.as_u8(), Ordering::Release);
1280            slot.replication_lag_bytes
1281                .store(u64::MAX, Ordering::Release);
1282        }
1283        drop(topology);
1284        let count = removed.len();
1285        drop(removed);
1286        if count > 0 {
1287            self.reaped_connections
1288                .fetch_add(count as u64, Ordering::AcqRel);
1289            self.record_connections_reaped(count);
1290        }
1291        count
1292    }
1293}
1294
1295/// Owned lifecycle handle for an active pool health-prober task.
1296///
1297/// Dropping the handle cancels the task. Use [`Self::shutdown`] to cancel and
1298/// wait for termination.
1299#[must_use = "dropping the handle immediately stops active health probing"]
1300pub struct HealthProberHandle {
1301    cancel: watch::Sender<bool>,
1302    task: Option<JoinHandle<()>>,
1303}
1304
1305impl HealthProberHandle {
1306    /// Cancel the prober and wait for its task to finish.
1307    pub async fn shutdown(mut self) {
1308        let _ = self.cancel.send(true);
1309        if let Some(task) = self.task.take() {
1310            let _ = task.await;
1311        }
1312    }
1313}
1314
1315impl Drop for HealthProberHandle {
1316    fn drop(&mut self) {
1317        let _ = self.cancel.send(true);
1318        if let Some(task) = &self.task {
1319            task.abort();
1320        }
1321    }
1322}
1323
1324/// Owned lifecycle handle for an idle-connection reaper task.
1325///
1326/// Dropping the handle cancels the task. Use [`Self::shutdown`] to cancel and
1327/// wait for termination.
1328#[must_use = "dropping the handle immediately stops idle-connection reaping"]
1329pub struct IdleReaperHandle {
1330    cancel: watch::Sender<bool>,
1331    task: Option<JoinHandle<()>>,
1332}
1333
1334impl IdleReaperHandle {
1335    /// Cancel the reaper and wait for its task to finish.
1336    pub async fn shutdown(mut self) {
1337        let _ = self.cancel.send(true);
1338        if let Some(task) = self.task.take() {
1339            let _ = task.await;
1340        }
1341    }
1342}
1343
1344impl Drop for IdleReaperHandle {
1345    fn drop(&mut self) {
1346        let _ = self.cancel.send(true);
1347        if let Some(task) = &self.task {
1348            task.abort();
1349        }
1350    }
1351}
1352
1353/// A pool of Redis connections that dispatches commands across them.
1354///
1355/// Generic over `S: RedisExecutor`, so it works with:
1356/// - `RedisConnection` (standalone)
1357/// - `ClusterConnection` (cluster -- each entry manages its own topology)
1358/// - `SentinelConnection` (sentinel -- each entry discovers its own master)
1359/// - `ResilientConnection` (standalone with auto-reconnect)
1360/// - Any custom type implementing `RedisExecutor`
1361///
1362/// The pool implements `Clone` via `Arc` for cross-task sharing and
1363/// implements `RedisExecutor` itself for composability.
1364///
1365/// # Concurrency
1366///
1367/// `ConnectionPool<S>` is `Clone + Send + Sync` when `S: Send`. All clones
1368/// share the same pool via `Arc`. Each individual connection is protected by
1369/// its own `Mutex`, so up to N commands can execute in parallel (one per
1370/// pooled connection). The dispatch strategy controls which connection handles
1371/// each command.
1372pub struct ConnectionPool<S> {
1373    inner: Arc<PoolInner<S>>,
1374}
1375
1376impl<S> Clone for ConnectionPool<S> {
1377    fn clone(&self) -> Self {
1378        Self {
1379            inner: Arc::clone(&self.inner),
1380        }
1381    }
1382}
1383
1384impl<S> ConnectionPool<S>
1385where
1386    S: Send + 'static,
1387{
1388    fn resolved_bounds(config: &PoolConfig) -> (usize, usize) {
1389        let min_size = config.min_size.unwrap_or(config.size);
1390        let max_size = config.max_size.unwrap_or(config.size);
1391        assert!(max_size > 0, "pool max_size must be at least 1");
1392        assert!(
1393            min_size <= max_size,
1394            "pool min_size ({min_size}) must not exceed max_size ({max_size})"
1395        );
1396        if let Some(idle_timeout) = config.idle_timeout {
1397            assert!(
1398                !idle_timeout.is_zero(),
1399                "pool idle_timeout must be non-zero"
1400            );
1401        }
1402        (min_size, max_size)
1403    }
1404
1405    fn from_initial(
1406        config: PoolConfig,
1407        initial: Vec<S>,
1408        min_size: usize,
1409        max_size: usize,
1410        factory: Option<Arc<dyn ErasedPoolFactory<Connection = S>>>,
1411    ) -> Self {
1412        assert!(initial.len() <= max_size);
1413        let now = now_millis();
1414        let mut connections = Vec::with_capacity(max_size);
1415        connections.extend(
1416            initial
1417                .into_iter()
1418                .map(|connection| PoolSlot::active(connection, now)),
1419        );
1420        connections.extend((connections.len()..max_size).map(|_| PoolSlot::vacant(now)));
1421        let health_check_interval_ms = config
1422            .health_check_interval
1423            .map(|duration| duration.as_millis().min(u64::MAX as u128) as u64)
1424            .unwrap_or(0);
1425        let idle_timeout_ms = config
1426            .idle_timeout
1427            .map(|duration| duration.as_millis().min(u64::MAX as u128) as u64);
1428
1429        Self {
1430            inner: Arc::new(PoolInner {
1431                name: config.name,
1432                connections,
1433                min_size,
1434                max_size,
1435                idle_timeout_ms,
1436                reaped_connections: AtomicU64::new(0),
1437                scale_lock: Mutex::new(()),
1438                topology_lock: RwLock::new(()),
1439                index: AtomicUsize::new(0),
1440                dispatch: config.dispatch,
1441                health_check_interval_ms,
1442                acquisition_timeout: config.acquisition_timeout,
1443                factory,
1444                metrics_recorder: config.metrics_recorder,
1445                admission_state: AtomicUsize::new(0),
1446            }),
1447        }
1448    }
1449
1450    /// Create a pool by calling a factory function `size` times.
1451    ///
1452    /// Each call to `factory` should return a new, independent connection.
1453    /// For cluster connections, each entry will discover its own topology.
1454    pub async fn connect<F, Fut>(size: usize, factory: F) -> Result<Self, RedisError>
1455    where
1456        F: Fn() -> Fut,
1457        Fut: Future<Output = Result<S, RedisError>>,
1458    {
1459        Self::connect_with_config(PoolConfig::default().size(size), factory).await
1460    }
1461
1462    /// Create a pool with custom configuration.
1463    pub async fn connect_with_config<F, Fut>(
1464        config: PoolConfig,
1465        factory: F,
1466    ) -> Result<Self, RedisError>
1467    where
1468        F: Fn() -> Fut,
1469        Fut: Future<Output = Result<S, RedisError>>,
1470    {
1471        let (min_size, max_size) = Self::resolved_bounds(&config);
1472        assert_eq!(
1473            min_size, max_size,
1474            "dynamic sizing requires connect_with_factory"
1475        );
1476        let mut connections = Vec::with_capacity(min_size);
1477        for _ in 0..min_size {
1478            let conn = factory().await?;
1479            connections.push(conn);
1480        }
1481        Ok(Self::from_initial(
1482            config,
1483            connections,
1484            min_size,
1485            max_size,
1486            None,
1487        ))
1488    }
1489
1490    /// Create a pool with custom configuration and a [`PoolFactory`].
1491    ///
1492    /// The factory is used both to build the initial connections and to
1493    /// replace any connection slot that fails a health-check PING. This
1494    /// ensures that a single dead connection does not permanently degrade
1495    /// pool capacity.
1496    pub async fn connect_with_factory<Fact>(
1497        config: PoolConfig,
1498        factory: Fact,
1499    ) -> Result<Self, RedisError>
1500    where
1501        Fact: PoolFactory<Connection = S>,
1502        S: RedisExecutor,
1503    {
1504        let (min_size, max_size) = Self::resolved_bounds(&config);
1505        let mut connections = Vec::with_capacity(min_size);
1506        for _ in 0..min_size {
1507            let conn = factory.create().await?;
1508            connections.push(conn);
1509        }
1510        Ok(Self::from_initial(
1511            config,
1512            connections,
1513            min_size,
1514            max_size,
1515            Some(Arc::new(factory)),
1516        ))
1517    }
1518
1519    /// Create a factory-backed pool without opening a connection.
1520    ///
1521    /// The first command creates the first slot; later acquisition contention
1522    /// grows the pool up to `config.max_size` (or `config.size` when omitted).
1523    /// The lazy pool's effective minimum is zero, regardless of
1524    /// `config.min_size`, so an explicitly spawned idle reaper may return it
1525    /// to zero while the application is inactive.
1526    pub fn connect_lazy<Fact>(config: PoolConfig, factory: Fact) -> Self
1527    where
1528        Fact: PoolFactory<Connection = S>,
1529        S: RedisExecutor,
1530    {
1531        let max_size = config.max_size.unwrap_or(config.size);
1532        assert!(max_size > 0, "pool max_size must be at least 1");
1533        if let Some(idle_timeout) = config.idle_timeout {
1534            assert!(
1535                !idle_timeout.is_zero(),
1536                "pool idle_timeout must be non-zero"
1537            );
1538        }
1539        Self::from_initial(config, Vec::new(), 0, max_size, Some(Arc::new(factory)))
1540    }
1541
1542    /// Build a pool from pre-created connections using the given [`PoolConfig`].
1543    ///
1544    /// Sizing fields are ignored here because a pool without a retained
1545    /// factory cannot restore reaped capacity. The pool is fixed at the number
1546    /// of supplied connections. Dispatch, health-check, acquisition-timeout,
1547    /// name, and metrics settings are applied normally.
1548    ///
1549    /// # Errors
1550    ///
1551    /// Returns an error if `connections` is empty.
1552    pub fn from_connections(connections: Vec<S>, config: PoolConfig) -> Result<Self, RedisError> {
1553        if connections.is_empty() {
1554            return Err(RedisError::InvalidUrl(
1555                "pool requires at least one connection".into(),
1556            ));
1557        }
1558
1559        let size = connections.len();
1560        Ok(Self::from_initial(config, connections, size, size, None))
1561    }
1562
1563    /// Returns the number of connections in the pool.
1564    pub fn size(&self) -> usize {
1565        self.inner.current_size()
1566    }
1567
1568    /// Returns the stable name used to identify this pool in metrics.
1569    pub fn name(&self) -> &str {
1570        &self.inner.name
1571    }
1572
1573    /// Returns the dispatch strategy.
1574    pub fn dispatch_strategy(&self) -> DispatchStrategy {
1575        self.inner.dispatch
1576    }
1577
1578    /// Returns a snapshot of current pool utilization.
1579    ///
1580    /// Reads the per-connection in-flight atomic counters to compute idle
1581    /// and active connection counts. This is a non-blocking, snapshot-in-
1582    /// time read — values may have changed by the time the caller acts on
1583    /// them.
1584    ///
1585    /// Useful for emitting `redis_pool_connections_active` /
1586    /// `redis_pool_connections_idle` Prometheus metrics.
1587    pub fn stats(&self) -> PoolStats {
1588        let mut total_inflight = 0usize;
1589        let mut max_inflight = 0usize;
1590        let mut idle_count = 0usize;
1591        let mut healthy_count = 0usize;
1592        let mut unhealthy_count = 0usize;
1593        let mut unknown_health_count = 0usize;
1594        let mut max_replication_lag_bytes = None;
1595        let topology = self.inner.topology_read();
1596        for slot in &self.inner.connections {
1597            if !slot.active.load(Ordering::Acquire) {
1598                continue;
1599            }
1600            let v = slot.inflight.load(Ordering::Relaxed);
1601            total_inflight += v;
1602            if v > max_inflight {
1603                max_inflight = v;
1604            }
1605            if v == 0 {
1606                idle_count += 1;
1607            }
1608            match PoolHealthState::from_u8(slot.health.load(Ordering::Acquire)) {
1609                PoolHealthState::Healthy => healthy_count += 1,
1610                PoolHealthState::Unhealthy => unhealthy_count += 1,
1611                PoolHealthState::Unknown => unknown_health_count += 1,
1612            }
1613            let lag = slot.replication_lag_bytes.load(Ordering::Acquire);
1614            if lag != u64::MAX {
1615                max_replication_lag_bytes =
1616                    Some(max_replication_lag_bytes.map_or(lag, |current: u64| current.max(lag)));
1617            }
1618        }
1619        let size = healthy_count + unhealthy_count + unknown_health_count;
1620        drop(topology);
1621        PoolStats {
1622            size,
1623            idle_count,
1624            total_inflight,
1625            max_inflight,
1626            min_size: self.inner.min_size,
1627            max_size: self.inner.max_size,
1628            healthy_count,
1629            unhealthy_count,
1630            unknown_health_count,
1631            max_replication_lag_bytes,
1632            reaped_connections: self.inner.reaped_connections.load(Ordering::Acquire),
1633        }
1634    }
1635
1636    /// Remove idle connections above the configured minimum immediately.
1637    ///
1638    /// This is a one-shot operation and does not spawn a task. It returns zero
1639    /// when no `idle_timeout` is configured or no eligible slot exists.
1640    pub async fn reap_idle_connections(&self) -> usize {
1641        self.inner.reap_idle_once().await
1642    }
1643
1644    /// Explicitly spawn periodic idle-connection reaping.
1645    ///
1646    /// Pool construction never starts this task. The first sweep runs
1647    /// immediately, then every `interval`. Dropping the returned handle stops
1648    /// the task.
1649    ///
1650    /// # Panics
1651    ///
1652    /// Panics when `interval` is zero, no idle timeout is configured, or this
1653    /// method is called outside a Tokio runtime.
1654    pub fn spawn_idle_reaper(&self, interval: Duration) -> IdleReaperHandle {
1655        assert!(!interval.is_zero(), "idle reaper interval must be non-zero");
1656        assert!(
1657            self.inner.idle_timeout_ms.is_some(),
1658            "spawn_idle_reaper requires PoolConfig::idle_timeout"
1659        );
1660        let pool = self.clone();
1661        let (cancel, mut cancelled) = watch::channel(false);
1662        let task = tokio::spawn(async move {
1663            let mut ticker = tokio::time::interval(interval);
1664            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1665            loop {
1666                tokio::select! {
1667                    changed = cancelled.changed() => {
1668                        if changed.is_err() || *cancelled.borrow() {
1669                            break;
1670                        }
1671                    }
1672                    _ = ticker.tick() => {
1673                        if pool.is_closed() {
1674                            break;
1675                        }
1676                        pool.reap_idle_connections().await;
1677                    }
1678                }
1679            }
1680        });
1681        IdleReaperHandle {
1682            cancel,
1683            task: Some(task),
1684        }
1685    }
1686
1687    /// Returns `true` once [`close`](Self::close) has been called on this pool
1688    /// or any of its clones.
1689    pub fn is_closed(&self) -> bool {
1690        self.inner.admission_state.load(Ordering::Acquire) & POOL_CLOSED_BIT != 0
1691    }
1692
1693    /// Gracefully close the pool, draining in-flight commands.
1694    ///
1695    /// Flips a shared "closed" bit so that every clone of this pool rejects new
1696    /// commands with [`RedisError::ConnectionClosed`], then waits for all
1697    /// in-flight commands to finish before returning. Each accepted command
1698    /// atomically increments a pool-wide operation count while confirming the
1699    /// pool is open, so closing and admission have one unambiguous order.
1700    ///
1701    /// This is the SIGTERM drain path: stop accepting work, let outstanding
1702    /// commands complete, then exit. It does not itself close the underlying
1703    /// connections -- dropping the pool (and its last clone) releases them --
1704    /// but it guarantees no command is mid-flight when the connections are
1705    /// dropped.
1706    ///
1707    /// Because the state is shared through the pool's `Arc`, calling `close` on
1708    /// one clone drains and closes the pool seen by every clone. Commands
1709    /// admitted before the close transition finish normally; commands racing
1710    /// after it are rejected.
1711    ///
1712    /// # Example
1713    ///
1714    /// ```no_run
1715    /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
1716    /// use redis_tower::{ConnectionPool, RedisConnection};
1717    ///
1718    /// let pool =
1719    ///     ConnectionPool::connect(4, || RedisConnection::connect("127.0.0.1:6379")).await?;
1720    ///
1721    /// // On shutdown (SIGTERM, ctrl-c, ...): stop accepting new work and drain
1722    /// // the pool before exit.
1723    /// pool.close().await;
1724    /// # Ok(())
1725    /// # }
1726    /// ```
1727    pub async fn close(self) {
1728        // Atomically close admission while preserving the accepted-operation
1729        // count. A racing reservation is therefore either included in the
1730        // count or observes the closed bit and is rejected.
1731        self.inner
1732            .admission_state
1733            .fetch_or(POOL_CLOSED_BIT, Ordering::AcqRel);
1734
1735        // A closed pool accepts no new work, so the active portion of the
1736        // state is monotonically non-increasing and eventually reaches zero.
1737        loop {
1738            if self.inner.admission_state.load(Ordering::Acquire) & POOL_ACTIVE_MASK == 0 {
1739                break;
1740            }
1741            tokio::time::sleep(Duration::from_millis(1)).await;
1742        }
1743    }
1744
1745    /// Select and reserve the next connection based on dispatch strategy.
1746    ///
1747    /// The returned RAII guard releases the in-flight count on drop, including
1748    /// when an execution future is cancelled at an await point.
1749    async fn next_index(
1750        &self,
1751        budget: AcquisitionBudget,
1752    ) -> Result<InflightReservation<'_, S>, RedisError> {
1753        self.inner.reserve_next(budget).await
1754    }
1755}
1756
1757impl<S> ConnectionPool<S>
1758where
1759    S: RedisExecutor + Send + 'static,
1760{
1761    /// Explicitly spawn active PING probing with an owned lifecycle handle.
1762    ///
1763    /// Pool construction never starts a prober. The first sweep runs
1764    /// immediately. Dropping the returned handle stops the task.
1765    pub fn spawn_health_prober(&self, config: HealthProberConfig) -> HealthProberHandle {
1766        self.spawn_health_prober_with(config, PingHealthProbe)
1767    }
1768
1769    /// Explicitly spawn active probing with a custom [`HealthProbe`].
1770    ///
1771    /// Results are observational: they update [`PoolStats`] and
1772    /// [`MetricsRecorder`] without silently changing dispatch policy.
1773    ///
1774    /// # Panics
1775    ///
1776    /// Panics when the interval or timeout is zero, or when called outside a
1777    /// Tokio runtime.
1778    pub fn spawn_health_prober_with<P>(
1779        &self,
1780        config: HealthProberConfig,
1781        probe: P,
1782    ) -> HealthProberHandle
1783    where
1784        P: HealthProbe<S>,
1785    {
1786        assert!(
1787            !config.interval.is_zero(),
1788            "health prober interval must be non-zero"
1789        );
1790        assert!(
1791            !config.timeout.is_zero(),
1792            "health prober timeout must be non-zero"
1793        );
1794        let pool = self.clone();
1795        let probe = Arc::new(probe);
1796        let kind = probe.kind();
1797        let (cancel, mut cancelled) = watch::channel(false);
1798        let task = tokio::spawn(async move {
1799            let mut ticker = tokio::time::interval(config.interval);
1800            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1801            loop {
1802                tokio::select! {
1803                    changed = cancelled.changed() => {
1804                        if changed.is_err() || *cancelled.borrow() {
1805                            break;
1806                        }
1807                    }
1808                    _ = ticker.tick() => {
1809                        if pool.is_closed() {
1810                            break;
1811                        }
1812                        let active: Vec<usize> = {
1813                            let _topology = pool.inner.topology_read();
1814                            pool.inner.connections.iter().enumerate()
1815                                .filter_map(|(index, slot)| {
1816                                    slot.active.load(Ordering::Acquire).then_some(index)
1817                                })
1818                                .collect()
1819                        };
1820                        for index in active {
1821                            if *cancelled.borrow() {
1822                                break;
1823                            }
1824                            // Admission is linearized with close(): a probe
1825                            // that starts first is drained, while a probe that
1826                            // loses the race sees the closed bit and the task
1827                            // exits without touching a connection.
1828                            let admission = match PoolAdmission::try_new(
1829                                &pool.inner.admission_state,
1830                            ) {
1831                                Ok(admission) => admission,
1832                                Err(_) => break,
1833                            };
1834                            let slot = &pool.inner.connections[index];
1835                            if !slot.active.load(Ordering::Acquire) {
1836                                continue;
1837                            }
1838                            // Active probing is observational and must never
1839                            // turn ordinary command contention into a health
1840                            // failure. Busy slots are retried on the next
1841                            // sweep instead of spending the probe timeout on
1842                            // their mutex.
1843                            let Ok(mut connection) = slot.connection.try_lock() else {
1844                                continue;
1845                            };
1846                            let Some(connection) = connection.as_mut() else {
1847                                continue;
1848                            };
1849                            let started = Instant::now();
1850                            let outcome = tokio::time::timeout(
1851                                config.timeout,
1852                                probe.probe(connection),
1853                            )
1854                            .await;
1855                            let result = match outcome {
1856                                Ok(Ok(result)) => result,
1857                                Ok(Err(_)) | Err(_) => HealthProbeResult::unhealthy(),
1858                            };
1859                            // Reaping may deactivate a slot while this task is
1860                            // waiting for its mutex. Do not publish a false
1861                            // failure for an intentionally removed connection.
1862                            if !slot.active.load(Ordering::Acquire) {
1863                                continue;
1864                            }
1865                            slot.health.store(
1866                                if result.healthy {
1867                                    PoolHealthState::Healthy.as_u8()
1868                                } else {
1869                                    PoolHealthState::Unhealthy.as_u8()
1870                                },
1871                                Ordering::Release,
1872                            );
1873                            slot.replication_lag_bytes.store(
1874                                result.replication_lag_bytes.unwrap_or(u64::MAX),
1875                                Ordering::Release,
1876                            );
1877                            if !result.healthy {
1878                                pool.inner.record_health_check_failed();
1879                            }
1880                            pool.inner.record_health_probe(
1881                                kind,
1882                                started.elapsed(),
1883                                result.healthy,
1884                                result.replication_lag_bytes,
1885                            );
1886                            drop(admission);
1887                        }
1888                    }
1889                }
1890            }
1891        });
1892        HealthProberHandle {
1893            cancel,
1894            task: Some(task),
1895        }
1896    }
1897}
1898
1899impl ConnectionPool<RedisConnection> {
1900    /// Send `AUTH` with `credentials` to every active pool connection.
1901    ///
1902    /// Slots are locked one at a time, so a credential update never races a
1903    /// command on the same socket. Other slots remain available while the
1904    /// update progresses. A connection created concurrently through a
1905    /// provider-backed factory fetches the provider's current credentials and
1906    /// therefore does not need to be revisited by this pass.
1907    ///
1908    /// If a slot rejects the replacement, this method returns the first error
1909    /// after attempting every active slot but leaves failed slots installed
1910    /// with their previous Redis identity. Call [`Self::close`] and rebuild the
1911    /// pool through its provider-backed factory when rotation must fail closed.
1912    pub async fn reauthenticate_all(&self, credentials: &Credentials) -> Result<(), RedisError> {
1913        let mut first_error = None;
1914        for slot in &self.inner.connections {
1915            if !slot.active.load(Ordering::Acquire) {
1916                continue;
1917            }
1918            let mut connection = slot.connection.lock().await;
1919            if !slot.active.load(Ordering::Acquire) {
1920                continue;
1921            }
1922            let Some(connection) = connection.as_mut() else {
1923                continue;
1924            };
1925            if let Err(error) = connection.execute(credentials.auth_command()).await
1926                && first_error.is_none()
1927            {
1928                first_error = Some(error);
1929            }
1930        }
1931        match first_error {
1932            Some(error) => Err(error),
1933            None => Ok(()),
1934        }
1935    }
1936
1937    /// Apply every credential emitted by `provider` to all active slots.
1938    ///
1939    /// Dropping the returned handle stops future updates without closing the
1940    /// pool. Use the same provider in the pool's
1941    /// [`CredentialConnectionFactory`](crate::CredentialConnectionFactory) so
1942    /// replacement and newly grown slots authenticate with current material.
1943    /// Rejected updates are logged without their error text and later updates
1944    /// are still consumed; failed slots are not automatically removed. A
1945    /// fail-closed owner must close and rebuild the pool before routing more
1946    /// application traffic.
1947    pub fn spawn_credential_reauthentication(
1948        &self,
1949        provider: Arc<dyn StreamingCredentialProvider>,
1950    ) -> CredentialReauthenticationHandle {
1951        let pool = self.clone();
1952        spawn_reauthentication_task(provider, move |credentials| {
1953            let pool = pool.clone();
1954            async move { pool.reauthenticate_all(&credentials).await }
1955        })
1956    }
1957}
1958
1959impl<S> RedisExecutor for ConnectionPool<S>
1960where
1961    S: RedisExecutor + Send + 'static,
1962{
1963    fn execute<Cmd: Command>(
1964        &mut self,
1965        cmd: Cmd,
1966    ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
1967        let inner = Arc::clone(&self.inner);
1968        async move {
1969            let deadline = cmd.deadline();
1970            let budget = AcquisitionBudget::new(inner.acquisition_timeout, deadline);
1971            // Keep reservation creation inside the async body. Constructing a
1972            // trait-method future must not reserve a slot before that future is
1973            // first polled. Admission and the closed check are one atomic step.
1974            let mut reservation = inner.reserve_next(budget).await?;
1975            // Acquire a connection, scanning for a free slot first to avoid
1976            // head-of-line blocking on a busy one, then falling back to an
1977            // awaited (optionally timed) lock on the preferred slot.
1978            let (idx, mut conn) = inner.acquire(&mut reservation, budget).await?;
1979            let connection = conn.as_mut().ok_or(RedisError::ConnectionClosed)?;
1980
1981            // Lazy health check: PING if idle beyond the threshold.
1982            // Gate the syscall behind the interval check to avoid calling
1983            // SystemTime::now() on every execute when health checks are disabled.
1984            if inner.health_check_interval_ms > 0 {
1985                let last = inner.connections[idx].last_used.load(Ordering::Acquire);
1986                let now = now_millis();
1987                if now.saturating_sub(last) >= inner.health_check_interval_ms
1988                    && let Err(ping_err) = connection.execute(Ping::new()).await
1989                {
1990                    inner.record_health_check_failed();
1991                    // PING failed. Attempt to replace the dead slot via the factory.
1992                    if let Some(ref factory) = inner.factory {
1993                        match factory.create().await {
1994                            Ok(fresh) => {
1995                                *connection = fresh;
1996                                inner.connections[idx]
1997                                    .last_used
1998                                    .store(now_millis(), Ordering::Release);
1999                                inner.connections[idx]
2000                                    .health
2001                                    .store(PoolHealthState::Unknown.as_u8(), Ordering::Release);
2002                                inner.record_connection_replaced();
2003                            }
2004                            Err(replace_err) => {
2005                                drop(conn);
2006                                return Err(replace_err);
2007                            }
2008                        }
2009                    } else {
2010                        drop(conn);
2011                        return Err(ping_err);
2012                    }
2013                }
2014            }
2015
2016            let result = connection.execute(cmd).await;
2017            if inner.health_check_interval_ms > 0 || inner.idle_timeout_ms.is_some() {
2018                inner.connections[idx]
2019                    .last_used
2020                    .store(now_millis(), Ordering::Release);
2021            }
2022            drop(conn);
2023            result
2024        }
2025    }
2026}
2027
2028// Also implement for &ConnectionPool so it can be used without mut
2029// (the pool handles interior mutability via per-connection Mutex).
2030impl<S> ConnectionPool<S>
2031where
2032    S: RedisExecutor + Send + 'static,
2033{
2034    /// Execute a command through the pool.
2035    ///
2036    /// This is the primary API. The pool selects a connection via the
2037    /// configured dispatch strategy and executes the command on it. To avoid
2038    /// head-of-line blocking, if the strategy-selected connection is currently
2039    /// busy the pool first scans for any idle connection (via `try_lock`) and
2040    /// uses that instead, only awaiting a busy slot when every connection is in
2041    /// use.
2042    ///
2043    /// If `health_check_interval` is configured and the selected connection
2044    /// has been idle longer than the interval, a PING is sent first to
2045    /// verify the connection is alive. If the PING fails and the pool was
2046    /// built with a [`PoolFactory`] (via [`ConnectionPool::connect_with_factory`]),
2047    /// the dead slot is replaced with a fresh connection before the actual
2048    /// command is executed. If no factory is available, the PING error is
2049    /// returned to the caller.
2050    ///
2051    /// If `cmd` is wrapped in [`redis_tower_core::WithDeadline`], pool
2052    /// acquisition observes that same absolute deadline. The request deadline
2053    /// and [`PoolConfig::acquisition_timeout`] do not add together: whichever
2054    /// expires first determines the error.
2055    pub async fn execute<Cmd: Command>(&self, cmd: Cmd) -> Result<Cmd::Response, RedisError> {
2056        let deadline = cmd.deadline();
2057        let budget = AcquisitionBudget::new(self.inner.acquisition_timeout, deadline);
2058        let mut reservation = self.next_index(budget).await?;
2059        // Acquire a connection, scanning for a free slot first to avoid
2060        // head-of-line blocking on a busy one, then falling back to an awaited
2061        // (optionally timed) lock on the preferred slot.
2062        let (idx, mut conn) = self.inner.acquire(&mut reservation, budget).await?;
2063        let connection = conn.as_mut().ok_or(RedisError::ConnectionClosed)?;
2064
2065        // Lazy health check: PING if idle beyond the threshold.
2066        // Gate the syscall behind the interval check to avoid calling
2067        // SystemTime::now() on every execute when health checks are disabled.
2068        if self.inner.health_check_interval_ms > 0 {
2069            let last = self.inner.connections[idx]
2070                .last_used
2071                .load(Ordering::Acquire);
2072            let now = now_millis();
2073            if now.saturating_sub(last) >= self.inner.health_check_interval_ms
2074                && let Err(ping_err) = connection.execute(Ping::new()).await
2075            {
2076                self.inner.record_health_check_failed();
2077                // PING failed. Attempt to replace the dead slot via the factory.
2078                if let Some(ref factory) = self.inner.factory {
2079                    match factory.create().await {
2080                        Ok(fresh) => {
2081                            *connection = fresh;
2082                            self.inner.connections[idx]
2083                                .last_used
2084                                .store(now_millis(), Ordering::Release);
2085                            self.inner.connections[idx]
2086                                .health
2087                                .store(PoolHealthState::Unknown.as_u8(), Ordering::Release);
2088                            self.inner.record_connection_replaced();
2089                        }
2090                        Err(replace_err) => {
2091                            drop(conn);
2092                            return Err(replace_err);
2093                        }
2094                    }
2095                } else {
2096                    drop(conn);
2097                    return Err(ping_err);
2098                }
2099            }
2100        }
2101
2102        let result = connection.execute(cmd).await;
2103        if self.inner.health_check_interval_ms > 0 || self.inner.idle_timeout_ms.is_some() {
2104            self.inner.connections[idx]
2105                .last_used
2106                .store(now_millis(), Ordering::Release);
2107        }
2108        drop(conn);
2109        result
2110    }
2111
2112    /// Send a PING to every connection in the pool.
2113    ///
2114    /// Acquires each connection sequentially and sends `PING`. Returns
2115    /// `Ok(())` if all connections respond successfully. Returns the first
2116    /// error encountered if any connection is unhealthy.
2117    ///
2118    /// Useful for Kubernetes readiness probes and `/health` endpoints. For
2119    /// a fast single-connection liveness check, call
2120    /// [`execute`](ConnectionPool::execute) with [`Ping`]
2121    /// directly.
2122    pub async fn health_check(&self) -> Result<(), RedisError> {
2123        let _admission = PoolAdmission::try_new(&self.inner.admission_state)?;
2124        for slot in &self.inner.connections {
2125            if !slot.active.load(Ordering::Acquire) {
2126                continue;
2127            }
2128            let mut conn = slot.connection.lock().await;
2129            let Some(conn) = conn.as_mut() else {
2130                continue;
2131            };
2132            if let Err(error) = conn.execute(Ping::new()).await {
2133                self.inner.record_health_check_failed();
2134                return Err(error);
2135            }
2136        }
2137        Ok(())
2138    }
2139}
2140
2141#[cfg(test)]
2142mod tests {
2143    use super::*;
2144    use crate::metrics_layer::ErrorKind;
2145    use bytes::Bytes;
2146    use redis_tower_core::{Frame, WithDeadline};
2147    use std::collections::VecDeque;
2148    use std::sync::Mutex as StdMutex;
2149    use std::sync::atomic::AtomicUsize;
2150
2151    #[derive(Default)]
2152    struct CountingRecorder {
2153        acquisitions: StdMutex<Vec<(String, Duration, bool)>>,
2154        health_check_failures: AtomicUsize,
2155        connection_replacements: AtomicUsize,
2156        health_probes: StdMutex<Vec<(HealthProbeKind, bool, Option<u64>)>>,
2157        reaped_connections: AtomicUsize,
2158    }
2159
2160    impl CountingRecorder {
2161        fn acquisitions(&self) -> Vec<(String, Duration, bool)> {
2162            self.acquisitions.lock().unwrap().clone()
2163        }
2164    }
2165
2166    impl MetricsRecorder for CountingRecorder {
2167        fn command_completed(
2168            &self,
2169            _command: &str,
2170            _duration: Duration,
2171            _error: Option<ErrorKind>,
2172        ) {
2173        }
2174
2175        fn pool_acquisition_completed(&self, pool_name: &str, duration: Duration, timed_out: bool) {
2176            self.acquisitions
2177                .lock()
2178                .unwrap()
2179                .push((pool_name.to_owned(), duration, timed_out));
2180        }
2181
2182        fn pool_health_check_failed(&self, _pool_name: &str) {
2183            self.health_check_failures.fetch_add(1, Ordering::Relaxed);
2184        }
2185
2186        fn pool_connection_replaced(&self, _pool_name: &str) {
2187            self.connection_replacements.fetch_add(1, Ordering::Relaxed);
2188        }
2189
2190        fn pool_health_probe_completed(
2191            &self,
2192            _pool_name: &str,
2193            kind: HealthProbeKind,
2194            _duration: Duration,
2195            healthy: bool,
2196            replication_lag_bytes: Option<u64>,
2197        ) {
2198            self.health_probes
2199                .lock()
2200                .unwrap()
2201                .push((kind, healthy, replication_lag_bytes));
2202        }
2203
2204        fn pool_connections_reaped(&self, _pool_name: &str, count: usize) {
2205            self.reaped_connections.fetch_add(count, Ordering::Relaxed);
2206        }
2207    }
2208
2209    /// Mock connection for testing pool dispatch without Redis.
2210    struct MockConn {
2211        _id: usize,
2212        responses: tokio::sync::Mutex<VecDeque<Frame>>,
2213        call_count: AtomicUsize,
2214    }
2215
2216    impl MockConn {
2217        fn new(id: usize, responses: Vec<Frame>) -> Self {
2218            Self {
2219                _id: id,
2220                responses: tokio::sync::Mutex::new(VecDeque::from(responses)),
2221                call_count: AtomicUsize::new(0),
2222            }
2223        }
2224
2225        fn calls(&self) -> usize {
2226            self.call_count.load(Ordering::Relaxed)
2227        }
2228    }
2229
2230    impl RedisExecutor for MockConn {
2231        fn execute<Cmd: Command>(
2232            &mut self,
2233            cmd: Cmd,
2234        ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
2235            self.call_count.fetch_add(1, Ordering::Relaxed);
2236            let frame = self
2237                .responses
2238                .try_lock()
2239                .ok()
2240                .and_then(|mut q| q.pop_front())
2241                .unwrap_or(Frame::Null);
2242            async move { cmd.parse_response(frame) }
2243        }
2244    }
2245
2246    struct DelayedConn {
2247        delay: Duration,
2248        call_count: AtomicUsize,
2249    }
2250
2251    impl DelayedConn {
2252        fn new(delay: Duration) -> Self {
2253            Self {
2254                delay,
2255                call_count: AtomicUsize::new(0),
2256            }
2257        }
2258
2259        fn calls(&self) -> usize {
2260            self.call_count.load(Ordering::Relaxed)
2261        }
2262    }
2263
2264    impl RedisExecutor for DelayedConn {
2265        fn execute<Cmd: Command>(
2266            &mut self,
2267            cmd: Cmd,
2268        ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
2269            self.call_count.fetch_add(1, Ordering::Relaxed);
2270            let delay = self.delay;
2271            async move {
2272                tokio::time::sleep(delay).await;
2273                cmd.parse_response(Frame::SimpleString(Bytes::from("PONG")))
2274            }
2275        }
2276    }
2277
2278    #[derive(Clone)]
2279    struct DelayedFactory {
2280        delay: Duration,
2281        creates: Arc<AtomicUsize>,
2282    }
2283
2284    impl DelayedFactory {
2285        fn new(delay: Duration) -> Self {
2286            Self {
2287                delay,
2288                creates: Arc::new(AtomicUsize::new(0)),
2289            }
2290        }
2291
2292        fn creates(&self) -> usize {
2293            self.creates.load(Ordering::Acquire)
2294        }
2295    }
2296
2297    impl PoolFactory for DelayedFactory {
2298        type Connection = DelayedConn;
2299
2300        fn create(
2301            &self,
2302        ) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
2303            self.creates.fetch_add(1, Ordering::AcqRel);
2304            let delay = self.delay;
2305            Box::pin(async move { Ok(DelayedConn::new(delay)) })
2306        }
2307    }
2308
2309    #[derive(Clone)]
2310    struct SlowFactory {
2311        connect_delay: Duration,
2312        creates: Arc<AtomicUsize>,
2313    }
2314
2315    impl SlowFactory {
2316        fn new(connect_delay: Duration) -> Self {
2317            Self {
2318                connect_delay,
2319                creates: Arc::new(AtomicUsize::new(0)),
2320            }
2321        }
2322    }
2323
2324    impl PoolFactory for SlowFactory {
2325        type Connection = DelayedConn;
2326
2327        fn create(
2328            &self,
2329        ) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
2330            self.creates.fetch_add(1, Ordering::AcqRel);
2331            let delay = self.connect_delay;
2332            Box::pin(async move {
2333                tokio::time::sleep(delay).await;
2334                Ok(DelayedConn::new(Duration::ZERO))
2335            })
2336        }
2337    }
2338
2339    #[derive(Clone)]
2340    struct CancelThenConnectFactory {
2341        calls: Arc<AtomicUsize>,
2342        first_entered: Arc<tokio::sync::Notify>,
2343    }
2344
2345    impl PoolFactory for CancelThenConnectFactory {
2346        type Connection = MockConn;
2347
2348        fn create(
2349            &self,
2350        ) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
2351            let call = self.calls.fetch_add(1, Ordering::AcqRel);
2352            let entered = Arc::clone(&self.first_entered);
2353            Box::pin(async move {
2354                if call == 0 {
2355                    entered.notify_one();
2356                    futures::future::pending::<()>().await;
2357                }
2358                Ok(MockConn::new(
2359                    call,
2360                    vec![Frame::SimpleString(Bytes::from("PONG"))],
2361                ))
2362            })
2363        }
2364    }
2365
2366    /// Mock factory that hands out pre-built MockConn instances one at a time.
2367    #[derive(Clone)]
2368    struct MockFactory {
2369        conns: Arc<tokio::sync::Mutex<VecDeque<MockConn>>>,
2370        creates: Arc<AtomicUsize>,
2371    }
2372
2373    impl MockFactory {
2374        fn new(conns: Vec<MockConn>) -> Self {
2375            Self {
2376                conns: Arc::new(tokio::sync::Mutex::new(VecDeque::from(conns))),
2377                creates: Arc::new(AtomicUsize::new(0)),
2378            }
2379        }
2380
2381        fn creates(&self) -> usize {
2382            self.creates.load(Ordering::Acquire)
2383        }
2384    }
2385
2386    impl PoolFactory for MockFactory {
2387        type Connection = MockConn;
2388
2389        fn create(
2390            &self,
2391        ) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
2392            let conns = Arc::clone(&self.conns);
2393            let creates = Arc::clone(&self.creates);
2394            Box::pin(async move {
2395                creates.fetch_add(1, Ordering::AcqRel);
2396                conns
2397                    .lock()
2398                    .await
2399                    .pop_front()
2400                    .ok_or_else(|| RedisError::InvalidUrl("no more mock connections".into()))
2401            })
2402        }
2403    }
2404
2405    #[tokio::test]
2406    async fn pool_default_config() {
2407        let config = PoolConfig::default();
2408        assert_eq!(config.name, "redis-tower");
2409        assert_eq!(config.size, 4);
2410        assert!(matches!(config.dispatch, DispatchStrategy::RoundRobin));
2411        assert!(config.metrics_recorder.is_none());
2412    }
2413
2414    #[tokio::test]
2415    async fn pool_config_builder() {
2416        let recorder = Arc::new(CountingRecorder::default());
2417        let config = PoolConfig::default()
2418            .name("primary-cache")
2419            .size(8)
2420            .dispatch(DispatchStrategy::Random)
2421            .metrics_recorder(recorder);
2422        let debug = format!("{config:?}");
2423
2424        assert_eq!(config.name, "primary-cache");
2425        assert_eq!(config.size, 8);
2426        assert!(matches!(config.dispatch, DispatchStrategy::Random));
2427        assert!(config.metrics_recorder.is_some());
2428        assert!(debug.contains("primary-cache"));
2429        assert!(debug.contains("<recorder>"));
2430    }
2431
2432    #[tokio::test]
2433    async fn pool_from_connections() {
2434        let conns = vec![MockConn::new(0, vec![]), MockConn::new(1, vec![])];
2435        let pool =
2436            ConnectionPool::from_connections(conns, PoolConfig::default().name("read-replicas"))
2437                .unwrap();
2438        assert_eq!(pool.size(), 2);
2439        assert_eq!(pool.name(), "read-replicas");
2440    }
2441
2442    #[tokio::test]
2443    async fn pool_empty_connections_fails() {
2444        let result = ConnectionPool::<MockConn>::from_connections(vec![], PoolConfig::default());
2445        assert!(result.is_err());
2446    }
2447
2448    #[tokio::test]
2449    async fn pool_round_robin_distributes() {
2450        use redis_tower_commands::Ping;
2451
2452        let conns = vec![
2453            MockConn::new(
2454                0,
2455                vec![
2456                    Frame::SimpleString(Bytes::from("PONG")),
2457                    Frame::SimpleString(Bytes::from("PONG")),
2458                ],
2459            ),
2460            MockConn::new(
2461                1,
2462                vec![
2463                    Frame::SimpleString(Bytes::from("PONG")),
2464                    Frame::SimpleString(Bytes::from("PONG")),
2465                ],
2466            ),
2467        ];
2468
2469        let pool = ConnectionPool::from_connections(
2470            conns,
2471            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
2472        )
2473        .unwrap();
2474
2475        // 4 commands should distribute 2 to each connection.
2476        for _ in 0..4 {
2477            let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2478        }
2479
2480        // Check distribution via the atomic counter -- pool alternates.
2481        // Connection 0 got calls 0, 2; connection 1 got calls 1, 3.
2482        let c0 = pool.inner.connections[0].lock().await;
2483        let c1 = pool.inner.connections[1].lock().await;
2484        assert_eq!(c0.calls(), 2);
2485        assert_eq!(c1.calls(), 2);
2486    }
2487
2488    #[tokio::test]
2489    async fn pool_connect_factory() {
2490        let counter = Arc::new(AtomicUsize::new(0));
2491        let c = counter.clone();
2492
2493        let pool = ConnectionPool::connect(3, || {
2494            let c = c.clone();
2495            async move {
2496                let id = c.fetch_add(1, Ordering::Relaxed);
2497                Ok::<_, RedisError>(MockConn::new(id, vec![]))
2498            }
2499        })
2500        .await
2501        .unwrap();
2502
2503        assert_eq!(pool.size(), 3);
2504        assert_eq!(counter.load(Ordering::Relaxed), 3);
2505    }
2506
2507    #[tokio::test]
2508    async fn pool_clone_shares_state() {
2509        use redis_tower_commands::Ping;
2510
2511        let conns = vec![MockConn::new(
2512            0,
2513            vec![
2514                Frame::SimpleString(Bytes::from("PONG")),
2515                Frame::SimpleString(Bytes::from("PONG")),
2516            ],
2517        )];
2518
2519        let pool = ConnectionPool::from_connections(
2520            conns,
2521            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
2522        )
2523        .unwrap();
2524        let pool2 = pool.clone();
2525
2526        let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2527        let _: Bytes = pool2.execute(Ping::new()).await.unwrap();
2528
2529        let c0 = pool.inner.connections[0].lock().await;
2530        assert_eq!(c0.calls(), 2); // Both clones hit the same connection.
2531    }
2532
2533    #[tokio::test]
2534    async fn pool_random_dispatch() {
2535        use redis_tower_commands::Ping;
2536
2537        let mut conns = Vec::new();
2538        for i in 0..4 {
2539            conns.push(MockConn::new(
2540                i,
2541                (0..10)
2542                    .map(|_| Frame::SimpleString(Bytes::from("PONG")))
2543                    .collect(),
2544            ));
2545        }
2546
2547        let pool = ConnectionPool::from_connections(
2548            conns,
2549            PoolConfig::default().dispatch(DispatchStrategy::Random),
2550        )
2551        .unwrap();
2552
2553        for _ in 0..20 {
2554            let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2555        }
2556
2557        // All 20 calls should have been distributed (not all to one connection).
2558        let mut total = 0;
2559        for c in &pool.inner.connections {
2560            total += c.lock().await.calls();
2561        }
2562        assert_eq!(total, 20);
2563    }
2564
2565    #[tokio::test]
2566    async fn pool_execute_returns_correct_response() {
2567        use redis_tower_commands::Get;
2568
2569        let conns = vec![MockConn::new(
2570            0,
2571            vec![Frame::BulkString(Some(Bytes::from("hello")))],
2572        )];
2573
2574        let pool = ConnectionPool::from_connections(
2575            conns,
2576            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
2577        )
2578        .unwrap();
2579        let result: Option<Bytes> = pool.execute(Get::new("key")).await.unwrap();
2580        assert_eq!(result, Some(Bytes::from("hello")));
2581    }
2582
2583    #[tokio::test]
2584    async fn pool_propagates_errors() {
2585        use redis_tower_commands::Get;
2586
2587        let conns = vec![MockConn::new(
2588            0,
2589            vec![Frame::Error(Bytes::from("ERR something went wrong"))],
2590        )];
2591
2592        let pool = ConnectionPool::from_connections(
2593            conns,
2594            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
2595        )
2596        .unwrap();
2597        let result = pool.execute(Get::new("key")).await;
2598        assert!(result.is_err());
2599    }
2600
2601    #[tokio::test]
2602    async fn pool_least_connections_prefers_idle() {
2603        use redis_tower_commands::Ping;
2604
2605        // Connection 0 has 0 inflight, connection 1 has 0 inflight.
2606        // With LeastConnections, sequential calls should still distribute
2607        // since inflight is decremented after each completes.
2608        let conns = vec![
2609            MockConn::new(
2610                0,
2611                (0..10)
2612                    .map(|_| Frame::SimpleString(Bytes::from("PONG")))
2613                    .collect(),
2614            ),
2615            MockConn::new(
2616                1,
2617                (0..10)
2618                    .map(|_| Frame::SimpleString(Bytes::from("PONG")))
2619                    .collect(),
2620            ),
2621        ];
2622
2623        let pool = ConnectionPool::from_connections(
2624            conns,
2625            PoolConfig::default().dispatch(DispatchStrategy::LeastConnections),
2626        )
2627        .unwrap();
2628
2629        // Sequential calls -- all inflight counts are 0 after each completes,
2630        // so least-connections falls back to picking index 0 each time.
2631        for _ in 0..4 {
2632            let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2633        }
2634
2635        let c0 = pool.inner.connections[0].lock().await;
2636        let c1 = pool.inner.connections[1].lock().await;
2637        // In sequential mode, connection 0 always has the lowest (tied) count,
2638        // so it gets all calls.
2639        assert_eq!(c0.calls(), 4);
2640        assert_eq!(c1.calls(), 0);
2641    }
2642
2643    #[tokio::test]
2644    async fn pool_least_connections_inflight_incremented_by_next_index() {
2645        // Verify that next_index() atomically increments the inflight counter
2646        // so concurrent callers cannot all pick the same connection.
2647        let conns = vec![
2648            MockConn::new(
2649                0,
2650                (0..10)
2651                    .map(|_| Frame::SimpleString(Bytes::from("PONG")))
2652                    .collect(),
2653            ),
2654            MockConn::new(
2655                1,
2656                (0..10)
2657                    .map(|_| Frame::SimpleString(Bytes::from("PONG")))
2658                    .collect(),
2659            ),
2660        ];
2661
2662        let pool = ConnectionPool::from_connections(
2663            conns,
2664            PoolConfig::default().dispatch(DispatchStrategy::LeastConnections),
2665        )
2666        .unwrap();
2667
2668        // Both start at 0. First next_index() picks 0 and increments it.
2669        let reservation0 = pool
2670            .next_index(AcquisitionBudget::new(pool.inner.acquisition_timeout, None))
2671            .await
2672            .unwrap();
2673        assert_eq!(reservation0.index(), 0);
2674        assert_eq!(
2675            pool.inner.connections[0].inflight.load(Ordering::Acquire),
2676            1
2677        );
2678        assert_eq!(
2679            pool.inner.connections[1].inflight.load(Ordering::Acquire),
2680            0
2681        );
2682
2683        // Second call should now pick connection 1 (inflight 0 < 1).
2684        let reservation1 = pool
2685            .next_index(AcquisitionBudget::new(pool.inner.acquisition_timeout, None))
2686            .await
2687            .unwrap();
2688        assert_eq!(reservation1.index(), 1);
2689        assert_eq!(
2690            pool.inner.connections[1].inflight.load(Ordering::Acquire),
2691            1
2692        );
2693
2694        drop((reservation0, reservation1));
2695        assert_eq!(pool.stats().total_inflight, 0);
2696    }
2697
2698    #[tokio::test]
2699    async fn pool_inflight_counters_are_zero_after_completion() {
2700        use redis_tower_commands::Ping;
2701
2702        let conns = vec![MockConn::new(
2703            0,
2704            vec![Frame::SimpleString(Bytes::from("PONG"))],
2705        )];
2706
2707        let pool = ConnectionPool::from_connections(
2708            conns,
2709            PoolConfig::default().dispatch(DispatchStrategy::LeastConnections),
2710        )
2711        .unwrap();
2712        let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2713
2714        assert_eq!(
2715            pool.inner.connections[0].inflight.load(Ordering::Relaxed),
2716            0
2717        );
2718    }
2719
2720    #[tokio::test]
2721    async fn pool_health_check_config() {
2722        let config = PoolConfig::default().health_check_interval(Duration::from_secs(30));
2723        assert_eq!(config.health_check_interval, Some(Duration::from_secs(30)));
2724    }
2725
2726    #[tokio::test]
2727    async fn pool_health_check_pings_stale_connection() {
2728        use redis_tower_commands::Ping;
2729
2730        // Provide 2 PONG responses: one for the health check PING, one for the actual command.
2731        let conns = vec![MockConn::new(
2732            0,
2733            vec![
2734                Frame::SimpleString(Bytes::from("PONG")),
2735                Frame::SimpleString(Bytes::from("PONG")),
2736            ],
2737        )];
2738
2739        // Use a very short health check interval (1 ms) so it always triggers.
2740        let pool = ConnectionPool::from_connections(
2741            conns,
2742            PoolConfig::default()
2743                .dispatch(DispatchStrategy::RoundRobin)
2744                .health_check_interval(Duration::from_millis(1)),
2745        )
2746        .unwrap();
2747
2748        // Set last_used to 0 (epoch) so the connection appears stale.
2749        pool.inner.connections[0]
2750            .last_used
2751            .store(0, Ordering::Release);
2752
2753        let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2754
2755        // The connection should have received 2 calls: the health check PING + the actual PING.
2756        let c0 = pool.inner.connections[0].lock().await;
2757        assert_eq!(c0.calls(), 2);
2758    }
2759
2760    #[tokio::test]
2761    async fn pool_health_check_skips_fresh_connection() {
2762        use redis_tower_commands::Ping;
2763
2764        // Only provide 1 PONG response -- health check should NOT trigger.
2765        let conns = vec![MockConn::new(
2766            0,
2767            vec![Frame::SimpleString(Bytes::from("PONG"))],
2768        )];
2769
2770        // Use a very long health check interval so it never triggers.
2771        let pool = ConnectionPool::from_connections(
2772            conns,
2773            PoolConfig::default()
2774                .dispatch(DispatchStrategy::RoundRobin)
2775                .health_check_interval(Duration::from_secs(3600)),
2776        )
2777        .unwrap();
2778
2779        let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2780
2781        // Only 1 call -- no health check PING was sent.
2782        let c0 = pool.inner.connections[0].lock().await;
2783        assert_eq!(c0.calls(), 1);
2784    }
2785
2786    #[tokio::test]
2787    async fn pool_no_health_check_when_disabled() {
2788        use redis_tower_commands::Ping;
2789
2790        // Only 1 PONG response available.
2791        let conns = vec![MockConn::new(
2792            0,
2793            vec![Frame::SimpleString(Bytes::from("PONG"))],
2794        )];
2795
2796        // No health check interval set (default).
2797        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
2798
2799        // Set last_used to 0 so connection appears stale.
2800        pool.inner.connections[0]
2801            .last_used
2802            .store(0, Ordering::Release);
2803
2804        let _: Bytes = pool.execute(Ping::new()).await.unwrap();
2805
2806        // Only 1 call -- health check is disabled.
2807        let c0 = pool.inner.connections[0].lock().await;
2808        assert_eq!(c0.calls(), 1);
2809    }
2810
2811    /// Verify that a dead connection slot is replaced by the factory after a
2812    /// failed health-check PING (issue #339).
2813    ///
2814    /// Sequence:
2815    ///   1. Pool is built via `connect_with_factory`; the initial connection
2816    ///      is a MockConn that returns an error for its first call (the health-
2817    ///      check PING will fail).
2818    ///   2. The factory supplies a fresh MockConn with a PONG response.
2819    ///   3. `execute(Ping::new())` is called; the health check triggers (last_used
2820    ///      is set to 0 so the slot appears stale).
2821    ///   4. The dead PING fails → factory creates a replacement → command
2822    ///      succeeds against the fresh connection.
2823    #[tokio::test]
2824    async fn pool_health_check_dead_connection_replaced() {
2825        use redis_tower_commands::Ping;
2826
2827        let recorder = Arc::new(CountingRecorder::default());
2828        // The factory serves two connections in order:
2829        //   slot 0 (initial): dead — first call returns an error (simulates a
2830        //     stale/closed connection whose health-check PING will fail).
2831        //   replacement: healthy — returns PONG for the actual command.
2832        let factory = MockFactory::new(vec![
2833            MockConn::new(0, vec![Frame::Error(Bytes::from("ERR connection closed"))]),
2834            MockConn::new(1, vec![Frame::SimpleString(Bytes::from("PONG"))]),
2835        ]);
2836
2837        let mut pool = ConnectionPool::connect_with_factory(
2838            PoolConfig::default()
2839                .name("write-pool")
2840                .size(1)
2841                .health_check_interval(Duration::from_millis(1))
2842                .metrics_recorder(recorder.clone()),
2843            factory,
2844        )
2845        .await
2846        .unwrap();
2847
2848        // Make the connection appear stale so the health check triggers.
2849        pool.inner.connections[0]
2850            .last_used
2851            .store(0, Ordering::Release);
2852
2853        // The execute call should:
2854        //  1. Trigger the health check (stale threshold exceeded).
2855        //  2. The PING on the dead connection returns an error.
2856        //  3. The factory creates the fresh connection and replaces the slot.
2857        //  4. The actual Ping command is sent on the fresh connection and succeeds.
2858        let result: Bytes = RedisExecutor::execute(&mut pool, Ping::new())
2859            .await
2860            .unwrap();
2861        assert_eq!(result, "PONG");
2862        assert_eq!(recorder.health_check_failures.load(Ordering::Relaxed), 1);
2863        assert_eq!(recorder.connection_replacements.load(Ordering::Relaxed), 1);
2864    }
2865
2866    /// Verify that when a health check PING fails and no factory is available,
2867    /// the error is returned to the caller (original behaviour preserved).
2868    #[tokio::test]
2869    async fn pool_health_check_dead_no_factory_returns_error() {
2870        use redis_tower_commands::Ping;
2871
2872        let recorder = Arc::new(CountingRecorder::default());
2873        let dead_conn = MockConn::new(0, vec![Frame::Error(Bytes::from("ERR connection closed"))]);
2874
2875        // No factory — use from_connections.
2876        let pool = ConnectionPool::from_connections(
2877            vec![dead_conn],
2878            PoolConfig::default()
2879                .dispatch(DispatchStrategy::RoundRobin)
2880                .health_check_interval(Duration::from_millis(1))
2881                .metrics_recorder(recorder.clone()),
2882        )
2883        .unwrap();
2884
2885        pool.inner.connections[0]
2886            .last_used
2887            .store(0, Ordering::Release);
2888
2889        let result = pool.execute(Ping::new()).await;
2890        assert!(result.is_err(), "expected error when no factory present");
2891        assert_eq!(recorder.health_check_failures.load(Ordering::Relaxed), 1);
2892        assert_eq!(recorder.connection_replacements.load(Ordering::Relaxed), 0);
2893    }
2894
2895    #[tokio::test]
2896    async fn pool_failed_replacement_is_not_recorded() {
2897        use redis_tower_commands::Ping;
2898
2899        let recorder = Arc::new(CountingRecorder::default());
2900        // The sole factory connection is installed initially, leaving no fresh
2901        // connection available when its lazy health check subsequently fails.
2902        let factory = MockFactory::new(vec![MockConn::new(
2903            0,
2904            vec![Frame::Error(Bytes::from("ERR connection closed"))],
2905        )]);
2906        let pool = ConnectionPool::connect_with_factory(
2907            PoolConfig::default()
2908                .size(1)
2909                .health_check_interval(Duration::from_millis(1))
2910                .metrics_recorder(recorder.clone()),
2911            factory,
2912        )
2913        .await
2914        .unwrap();
2915        pool.inner.connections[0]
2916            .last_used
2917            .store(0, Ordering::Release);
2918
2919        let result: Result<Bytes, _> = pool.execute(Ping::new()).await;
2920
2921        assert!(result.is_err(), "replacement factory should be exhausted");
2922        assert_eq!(recorder.health_check_failures.load(Ordering::Relaxed), 1);
2923        assert_eq!(recorder.connection_replacements.load(Ordering::Relaxed), 0);
2924    }
2925
2926    /// When acquisition_timeout is set and all slots are busy,
2927    /// execute() returns PoolAcquisitionTimeout promptly.
2928    #[tokio::test]
2929    async fn pool_acquisition_timeout_fires_when_all_busy() {
2930        use redis_tower_commands::Ping;
2931        use std::time::Instant;
2932
2933        // Single-connection pool so it's easy to saturate.
2934        let conns = vec![MockConn::new(
2935            0,
2936            vec![Frame::SimpleString(Bytes::from("PONG"))],
2937        )];
2938
2939        let pool = ConnectionPool::from_connections(
2940            conns,
2941            PoolConfig::default().acquisition_timeout(Duration::from_millis(50)),
2942        )
2943        .unwrap();
2944
2945        // Hold the lock on slot 0 to simulate a busy connection.
2946        let _guard = pool.inner.connections[0].lock().await;
2947
2948        let start = Instant::now();
2949        let result = pool.execute(Ping::new()).await;
2950        let elapsed = start.elapsed();
2951
2952        assert!(
2953            matches!(result, Err(RedisError::PoolAcquisitionTimeout { .. })),
2954            "expected PoolAcquisitionTimeout, got {result:?}"
2955        );
2956        // Should return well under 1 second (timeout is 50 ms).
2957        assert!(elapsed < Duration::from_secs(1));
2958    }
2959
2960    #[tokio::test]
2961    async fn zero_acquisition_timeout_fails_fast_when_all_slots_are_busy() {
2962        let pool = ConnectionPool::from_connections(
2963            vec![MockConn::new(
2964                0,
2965                vec![Frame::SimpleString(Bytes::from("PONG"))],
2966            )],
2967            PoolConfig::default().acquisition_timeout(Duration::ZERO),
2968        )
2969        .unwrap();
2970        let _guard = pool.inner.connections[0].lock().await;
2971
2972        let result: Result<Bytes, _> = pool.execute(Ping::new()).await;
2973
2974        assert!(matches!(
2975            result,
2976            Err(RedisError::PoolAcquisitionTimeout {
2977                waited: Duration::ZERO,
2978                pool_size: 1,
2979            })
2980        ));
2981        assert_eq!(pool.stats().total_inflight, 0);
2982    }
2983
2984    #[tokio::test]
2985    async fn zero_acquisition_timeout_still_uses_an_immediately_available_slot() {
2986        let pool = ConnectionPool::from_connections(
2987            vec![MockConn::new(
2988                0,
2989                vec![Frame::SimpleString(Bytes::from("PONG"))],
2990            )],
2991            PoolConfig::default().acquisition_timeout(Duration::ZERO),
2992        )
2993        .unwrap();
2994
2995        let result: Bytes = pool.execute(Ping::new()).await.unwrap();
2996
2997        assert_eq!(result, "PONG");
2998    }
2999
3000    #[tokio::test]
3001    async fn request_deadline_bounds_pool_acquisition_without_static_timeout() {
3002        let recorder = Arc::new(CountingRecorder::default());
3003        let pool = ConnectionPool::from_connections(
3004            vec![MockConn::new(
3005                0,
3006                vec![Frame::SimpleString(Bytes::from("PONG"))],
3007            )],
3008            PoolConfig::default()
3009                .disable_acquisition_timeout()
3010                .metrics_recorder(recorder.clone()),
3011        )
3012        .unwrap();
3013        let _guard = pool.inner.connections[0].lock().await;
3014
3015        let result: Result<Bytes, _> = pool
3016            .execute(WithDeadline::after(Ping::new(), Duration::from_millis(25)))
3017            .await;
3018
3019        assert!(matches!(result, Err(RedisError::CommandTimeout)));
3020        assert_eq!(pool.stats().total_inflight, 0);
3021        let acquisitions = recorder.acquisitions();
3022        assert_eq!(acquisitions.len(), 1);
3023        assert!(acquisitions[0].2);
3024    }
3025
3026    #[tokio::test(start_paused = true)]
3027    async fn request_deadline_wins_when_lock_is_released_at_deadline() {
3028        let pool = ConnectionPool::from_connections(
3029            vec![MockConn::new(
3030                0,
3031                vec![Frame::SimpleString(Bytes::from("PONG"))],
3032            )],
3033            PoolConfig::default().disable_acquisition_timeout(),
3034        )
3035        .unwrap();
3036        let deadline = TokioInstant::now() + Duration::from_secs(1);
3037
3038        let holder_pool = pool.clone();
3039        let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
3040        let holder = tokio::spawn(async move {
3041            let guard = holder_pool.inner.connections[0].lock().await;
3042            locked_tx.send(()).unwrap();
3043            tokio::time::sleep_until(deadline).await;
3044            drop(guard);
3045        });
3046        locked_rx.await.unwrap();
3047
3048        let caller_pool = pool.clone();
3049        let caller = tokio::spawn(async move {
3050            caller_pool
3051                .execute(WithDeadline::new(Ping::new(), deadline))
3052                .await
3053        });
3054        tokio::task::yield_now().await;
3055        tokio::time::advance(Duration::from_secs(1)).await;
3056
3057        holder.await.unwrap();
3058        let result: Result<Bytes, _> = caller.await.unwrap();
3059        assert!(matches!(result, Err(RedisError::CommandTimeout)));
3060        assert_eq!(pool.inner.connections[0].lock().await.calls(), 0);
3061    }
3062
3063    #[tokio::test(start_paused = true)]
3064    async fn static_timeout_wins_when_lock_is_released_at_deadline() {
3065        let pool = ConnectionPool::from_connections(
3066            vec![MockConn::new(
3067                0,
3068                vec![Frame::SimpleString(Bytes::from("PONG"))],
3069            )],
3070            PoolConfig::default().acquisition_timeout(Duration::from_secs(1)),
3071        )
3072        .unwrap();
3073
3074        let holder_pool = pool.clone();
3075        let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
3076        let holder = tokio::spawn(async move {
3077            let guard = holder_pool.inner.connections[0].lock().await;
3078            locked_tx.send(()).unwrap();
3079            tokio::time::sleep(Duration::from_secs(1)).await;
3080            drop(guard);
3081        });
3082        locked_rx.await.unwrap();
3083
3084        let caller_pool = pool.clone();
3085        let caller = tokio::spawn(async move { caller_pool.execute(Ping::new()).await });
3086        tokio::task::yield_now().await;
3087        tokio::time::advance(Duration::from_secs(1)).await;
3088
3089        holder.await.unwrap();
3090        let result: Result<Bytes, _> = caller.await.unwrap();
3091        assert!(matches!(
3092            result,
3093            Err(RedisError::PoolAcquisitionTimeout {
3094                waited,
3095                pool_size: 1,
3096            }) if waited == Duration::from_secs(1)
3097        ));
3098        assert_eq!(pool.inner.connections[0].lock().await.calls(), 0);
3099    }
3100
3101    #[tokio::test]
3102    async fn static_pool_timeout_wins_when_earlier_than_request_deadline() {
3103        let pool = ConnectionPool::from_connections(
3104            vec![MockConn::new(
3105                0,
3106                vec![Frame::SimpleString(Bytes::from("PONG"))],
3107            )],
3108            PoolConfig::default().acquisition_timeout(Duration::from_millis(20)),
3109        )
3110        .unwrap();
3111        let _guard = pool.inner.connections[0].lock().await;
3112
3113        let result: Result<Bytes, _> = pool
3114            .execute(WithDeadline::after(Ping::new(), Duration::from_secs(1)))
3115            .await;
3116
3117        assert!(matches!(
3118            result,
3119            Err(RedisError::PoolAcquisitionTimeout { .. })
3120        ));
3121        assert_eq!(pool.stats().total_inflight, 0);
3122    }
3123
3124    #[tokio::test]
3125    async fn expired_request_does_not_execute_on_an_idle_pool() {
3126        let pool = ConnectionPool::from_connections(
3127            vec![MockConn::new(
3128                0,
3129                vec![Frame::SimpleString(Bytes::from("PONG"))],
3130            )],
3131            PoolConfig::default(),
3132        )
3133        .unwrap();
3134        let command =
3135            WithDeadline::new(Ping::new(), TokioInstant::now() - Duration::from_millis(1));
3136
3137        let result: Result<Bytes, _> = pool.execute(command).await;
3138
3139        assert!(matches!(result, Err(RedisError::CommandTimeout)));
3140        assert_eq!(pool.stats().total_inflight, 0);
3141        assert_eq!(pool.inner.connections[0].lock().await.calls(), 0);
3142    }
3143
3144    #[tokio::test]
3145    async fn one_deadline_budget_spans_pool_acquisition_and_execution() {
3146        use crate::{CommandTimeoutLayer, ExecutorService};
3147        use tower_layer::Layer;
3148        use tower_service::Service;
3149
3150        let pool = ConnectionPool::from_connections(
3151            vec![DelayedConn::new(Duration::from_millis(500))],
3152            PoolConfig::default().disable_acquisition_timeout(),
3153        )
3154        .unwrap();
3155
3156        // Occupy the only slot first, then release it after part of the
3157        // request's absolute budget has already been consumed.
3158        let holder_pool = pool.clone();
3159        let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
3160        let holder = tokio::spawn(async move {
3161            let _guard = holder_pool.inner.connections[0].lock().await;
3162            let _ = locked_tx.send(());
3163            tokio::time::sleep(Duration::from_millis(30)).await;
3164        });
3165        locked_rx.await.unwrap();
3166
3167        let mut service = CommandTimeoutLayer::new(Duration::from_secs(1))
3168            .with_request_deadlines()
3169            .layer(ExecutorService::new(pool.clone()));
3170        let result: Result<Bytes, _> = service
3171            .call(WithDeadline::after(Ping::new(), Duration::from_millis(200)))
3172            .await;
3173
3174        holder.await.unwrap();
3175        assert!(matches!(result, Err(RedisError::CommandTimeout)));
3176        assert_eq!(pool.stats().total_inflight, 0);
3177        assert_eq!(pool.inner.connections[0].lock().await.calls(), 1);
3178    }
3179
3180    #[cfg(unix)]
3181    #[tokio::test]
3182    async fn timed_out_pooled_connection_does_not_reuse_unread_response() {
3183        use crate::{CommandTimeoutLayer, ExecutorService};
3184        use futures::{SinkExt, StreamExt};
3185        use redis_tower_core::{RedisConnection, RedisStream};
3186        use redis_tower_protocol::RespCodec;
3187        use tokio::sync::oneshot;
3188        use tokio_util::codec::Framed;
3189        use tower_layer::Layer;
3190        use tower_service::Service;
3191
3192        let (client, server) = tokio::net::UnixStream::pair().unwrap();
3193        let (wire_tx, wire_rx) = oneshot::channel();
3194        let (release_tx, release_rx) = oneshot::channel();
3195        let (late_attempt_tx, late_attempt_rx) = oneshot::channel();
3196
3197        let server_task = tokio::spawn(async move {
3198            let mut framed = Framed::new(RedisStream::Unix(server), RespCodec::new());
3199            framed
3200                .next()
3201                .await
3202                .expect("client closed before first pooled command")
3203                .expect("client sent an invalid first pooled command");
3204            wire_tx.send(()).unwrap();
3205
3206            release_rx.await.unwrap();
3207            let _ = framed
3208                .send(Frame::SimpleString(Bytes::from_static(b"LATE")))
3209                .await;
3210            late_attempt_tx.send(()).unwrap();
3211
3212            match tokio::time::timeout(Duration::from_secs(1), framed.next()).await {
3213                Ok(None) | Ok(Some(Err(_))) => {}
3214                Ok(Some(Ok(frame))) => panic!("timed-out pool slot was reused: {frame:?}"),
3215                Err(_) => panic!("timed-out pool socket remained open"),
3216            }
3217        });
3218
3219        let connection = RedisConnection::from_stream(RedisStream::Unix(client));
3220        let pool = ConnectionPool::from_connections(
3221            vec![connection],
3222            PoolConfig::default().disable_acquisition_timeout(),
3223        )
3224        .unwrap();
3225        let mut service = CommandTimeoutLayer::new(Duration::from_secs(1))
3226            .with_request_deadlines()
3227            .layer(ExecutorService::new(pool.clone()));
3228        let mut first =
3229            Box::pin(service.call(WithDeadline::after(Ping::new(), Duration::from_millis(100))));
3230
3231        tokio::select! {
3232            result = &mut first => panic!("pooled call completed before reaching wire: {result:?}"),
3233            observed = wire_rx => observed.unwrap(),
3234        }
3235        assert!(matches!(first.await, Err(RedisError::CommandTimeout)));
3236
3237        release_tx.send(()).unwrap();
3238        late_attempt_rx.await.unwrap();
3239
3240        let successor: Result<Bytes, _> = pool.execute(Ping::new()).await;
3241        assert!(matches!(successor, Err(RedisError::ConnectionClosed)));
3242        server_task.await.unwrap();
3243    }
3244
3245    #[tokio::test]
3246    async fn pool_records_successful_and_timed_out_acquisitions() {
3247        use redis_tower_commands::Ping;
3248
3249        let recorder = Arc::new(CountingRecorder::default());
3250        let pool = ConnectionPool::from_connections(
3251            vec![MockConn::new(
3252                0,
3253                vec![Frame::SimpleString(Bytes::from("PONG"))],
3254            )],
3255            PoolConfig::default()
3256                .name("acquisition-test")
3257                .acquisition_timeout(Duration::from_millis(10))
3258                .metrics_recorder(recorder.clone()),
3259        )
3260        .unwrap();
3261
3262        let pong: Bytes = pool.execute(Ping::new()).await.unwrap();
3263        assert_eq!(pong, "PONG");
3264
3265        let _guard = pool.inner.connections[0].lock().await;
3266        let result: Result<Bytes, _> = pool.execute(Ping::new()).await;
3267        assert!(matches!(
3268            result,
3269            Err(RedisError::PoolAcquisitionTimeout { .. })
3270        ));
3271
3272        let acquisitions = recorder.acquisitions();
3273        assert_eq!(acquisitions.len(), 2);
3274        assert_eq!(acquisitions[0].0, "acquisition-test");
3275        assert!(!acquisitions[0].2);
3276        assert_eq!(acquisitions[1].0, "acquisition-test");
3277        assert!(acquisitions[1].1 >= Duration::from_millis(10));
3278        assert!(acquisitions[1].2);
3279        assert_eq!(
3280            pool.inner.connections[0].inflight.load(Ordering::Acquire),
3281            0
3282        );
3283    }
3284
3285    /// The default config now bounds the acquisition wait, so callers fail
3286    /// fast on a saturated pool instead of stalling forever.
3287    #[test]
3288    fn pool_default_config_bounds_acquisition_timeout() {
3289        let config = PoolConfig::default();
3290        assert_eq!(
3291            config.acquisition_timeout,
3292            Some(PoolConfig::DEFAULT_ACQUISITION_TIMEOUT),
3293        );
3294        assert_eq!(
3295            PoolConfig::DEFAULT_ACQUISITION_TIMEOUT,
3296            Duration::from_secs(5),
3297        );
3298        // disable_acquisition_timeout opts back into the unbounded wait.
3299        assert_eq!(
3300            PoolConfig::default()
3301                .disable_acquisition_timeout()
3302                .acquisition_timeout,
3303            None,
3304        );
3305    }
3306
3307    /// With the acquisition timeout disabled, execute() blocks until the lock
3308    /// is available rather than timing out — the opt-in unbounded behavior.
3309    #[tokio::test]
3310    async fn pool_no_acquisition_timeout_blocks_until_available() {
3311        use redis_tower_commands::Ping;
3312
3313        let conns = vec![MockConn::new(
3314            0,
3315            vec![Frame::SimpleString(Bytes::from("PONG"))],
3316        )];
3317
3318        let pool = ConnectionPool::from_connections(
3319            conns,
3320            PoolConfig::default().disable_acquisition_timeout(),
3321        )
3322        .unwrap();
3323
3324        // Release the lock after a short delay on a background task.
3325        let pool2 = pool.clone();
3326        tokio::spawn(async move {
3327            let _guard = pool2.inner.connections[0].lock().await;
3328            tokio::time::sleep(Duration::from_millis(20)).await;
3329            // guard drops here, releasing the lock
3330        });
3331
3332        // Give the spawned task time to acquire the lock.
3333        tokio::time::sleep(Duration::from_millis(5)).await;
3334
3335        // execute() should block until the background task releases, then succeed.
3336        let result: Bytes = pool.execute(Ping::new()).await.unwrap();
3337        assert_eq!(result, "PONG");
3338    }
3339
3340    /// Head-of-line-blocking avoidance: when the strategy-preferred connection
3341    /// is busy but another sits idle, execute() dispatches to the idle one via
3342    /// the try_lock scan instead of queuing behind the busy slot.
3343    #[tokio::test]
3344    async fn pool_avoids_head_of_line_blocking() {
3345        use redis_tower_commands::Ping;
3346        use std::time::Instant;
3347
3348        let conns = vec![
3349            MockConn::new(0, vec![Frame::SimpleString(Bytes::from("PONG"))]),
3350            MockConn::new(1, vec![Frame::SimpleString(Bytes::from("PONG"))]),
3351        ];
3352
3353        // RoundRobin with a fresh index counter: the first execute prefers
3354        // slot 0.
3355        let pool = ConnectionPool::from_connections(
3356            conns,
3357            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
3358        )
3359        .unwrap();
3360
3361        // Simulate a long-running command holding the preferred slot (0).
3362        let busy = pool.inner.connections[0].lock().await;
3363
3364        // execute() prefers slot 0 (busy) but should fall through to the idle
3365        // slot 1 and return promptly rather than waiting on the held lock.
3366        let start = Instant::now();
3367        let pong: Bytes = pool.execute(Ping::new()).await.unwrap();
3368        let elapsed = start.elapsed();
3369
3370        assert_eq!(pong, "PONG");
3371        assert!(
3372            elapsed < Duration::from_millis(500),
3373            "execute should not block on the busy slot, took {elapsed:?}"
3374        );
3375
3376        drop(busy);
3377
3378        // The command ran on the idle slot 1, not the busy slot 0.
3379        assert_eq!(pool.inner.connections[1].lock().await.calls(), 1);
3380        assert_eq!(pool.inner.connections[0].lock().await.calls(), 0);
3381        // Inflight accounting was moved off the preferred slot and released.
3382        assert_eq!(
3383            pool.inner.connections[0].inflight.load(Ordering::Acquire),
3384            0
3385        );
3386        assert_eq!(
3387            pool.inner.connections[1].inflight.load(Ordering::Acquire),
3388            0
3389        );
3390    }
3391
3392    #[tokio::test]
3393    async fn pool_stats_all_idle_on_fresh_pool() {
3394        let conns = vec![
3395            MockConn::new(0, vec![]),
3396            MockConn::new(1, vec![]),
3397            MockConn::new(2, vec![]),
3398        ];
3399
3400        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
3401        let stats = pool.stats();
3402
3403        assert_eq!(stats.size, 3);
3404        assert_eq!(stats.idle_count, 3);
3405        assert_eq!(stats.total_inflight, 0);
3406        assert_eq!(stats.max_inflight, 0);
3407    }
3408
3409    #[tokio::test]
3410    async fn pool_stats_size_matches_pool_size() {
3411        let conns = vec![MockConn::new(0, vec![]), MockConn::new(1, vec![])];
3412
3413        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
3414
3415        assert_eq!(pool.stats().size, pool.size());
3416    }
3417
3418    #[tokio::test]
3419    async fn pool_health_check_all_healthy() {
3420        use redis_tower_commands::Ping;
3421
3422        // 2 connections, each with a PONG response for the health check.
3423        let conns = vec![
3424            MockConn::new(0, vec![Frame::SimpleString(Bytes::from("PONG"))]),
3425            MockConn::new(1, vec![Frame::SimpleString(Bytes::from("PONG"))]),
3426        ];
3427
3428        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
3429        let result = pool.health_check().await;
3430        assert!(result.is_ok());
3431
3432        // Both connections should have received one PING call each.
3433        let c0 = pool.inner.connections[0].lock().await;
3434        let c1 = pool.inner.connections[1].lock().await;
3435        assert_eq!(c0.calls(), 1);
3436        assert_eq!(c1.calls(), 1);
3437
3438        // Suppress unused import warning in test.
3439        let _: std::marker::PhantomData<Ping> = std::marker::PhantomData;
3440    }
3441
3442    #[tokio::test]
3443    async fn pool_health_check_returns_first_error() {
3444        let recorder = Arc::new(CountingRecorder::default());
3445        // Connection 0 returns an error frame (no healthy response).
3446        let conns = vec![
3447            MockConn::new(0, vec![Frame::Error(Bytes::from("ERR connection dead"))]),
3448            MockConn::new(1, vec![Frame::SimpleString(Bytes::from("PONG"))]),
3449        ];
3450
3451        let pool = ConnectionPool::from_connections(
3452            conns,
3453            PoolConfig::default().metrics_recorder(recorder.clone()),
3454        )
3455        .unwrap();
3456        let result = pool.health_check().await;
3457        assert!(result.is_err());
3458        assert_eq!(recorder.health_check_failures.load(Ordering::Relaxed), 1);
3459        assert_eq!(recorder.connection_replacements.load(Ordering::Relaxed), 0);
3460    }
3461
3462    /// A connection whose `execute` blocks until the test releases it, used to
3463    /// hold a command in-flight while `close()` is observed.
3464    struct BlockingConn {
3465        started: Arc<tokio::sync::Notify>,
3466        release: Arc<tokio::sync::Notify>,
3467    }
3468
3469    impl RedisExecutor for BlockingConn {
3470        fn execute<Cmd: Command>(
3471            &mut self,
3472            cmd: Cmd,
3473        ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
3474            let started = Arc::clone(&self.started);
3475            let release = Arc::clone(&self.release);
3476            async move {
3477                started.notify_one();
3478                release.notified().await;
3479                cmd.parse_response(Frame::SimpleString(Bytes::from("PONG")))
3480            }
3481        }
3482    }
3483
3484    #[tokio::test]
3485    async fn pool_unpolled_trait_future_never_reserves_connection() {
3486        let pool = ConnectionPool::from_connections(
3487            vec![MockConn::new(
3488                0,
3489                vec![Frame::SimpleString(Bytes::from("PONG"))],
3490            )],
3491            PoolConfig::default(),
3492        )
3493        .unwrap();
3494        let mut executor_view = pool.clone();
3495
3496        let future = RedisExecutor::execute(&mut executor_view, Ping::new());
3497
3498        assert_eq!(pool.stats().total_inflight, 0);
3499        drop(future);
3500        assert_eq!(pool.stats().total_inflight, 0);
3501        tokio::time::timeout(Duration::from_secs(1), pool.close())
3502            .await
3503            .expect("an unpolled command future must not prevent pool close");
3504    }
3505
3506    #[tokio::test]
3507    async fn pool_abort_while_waiting_releases_reservation() {
3508        let pool = ConnectionPool::from_connections(
3509            vec![MockConn::new(
3510                0,
3511                vec![Frame::SimpleString(Bytes::from("PONG"))],
3512            )],
3513            PoolConfig::default().disable_acquisition_timeout(),
3514        )
3515        .unwrap();
3516        let connection_guard = pool.inner.connections[0].lock().await;
3517        let task_pool = pool.clone();
3518        let task = tokio::spawn(async move { task_pool.execute(Ping::new()).await });
3519
3520        tokio::time::timeout(Duration::from_secs(1), async {
3521            while pool.stats().total_inflight == 0 {
3522                tokio::task::yield_now().await;
3523            }
3524        })
3525        .await
3526        .expect("command should reserve a slot before waiting for its lock");
3527        assert_eq!(pool.stats().total_inflight, 1);
3528
3529        task.abort();
3530        assert!(task.await.unwrap_err().is_cancelled());
3531        assert_eq!(pool.stats().total_inflight, 0);
3532        drop(connection_guard);
3533
3534        tokio::time::timeout(Duration::from_secs(1), pool.close())
3535            .await
3536            .expect("an aborted acquisition must not prevent pool close");
3537    }
3538
3539    #[tokio::test]
3540    async fn pool_abort_during_command_releases_reservation() {
3541        let started = Arc::new(tokio::sync::Notify::new());
3542        let release = Arc::new(tokio::sync::Notify::new());
3543        let pool = ConnectionPool::from_connections(
3544            vec![BlockingConn {
3545                started: Arc::clone(&started),
3546                release,
3547            }],
3548            PoolConfig::default(),
3549        )
3550        .unwrap();
3551        let mut task_pool = pool.clone();
3552        let task =
3553            tokio::spawn(async move { RedisExecutor::execute(&mut task_pool, Ping::new()).await });
3554
3555        started.notified().await;
3556        assert_eq!(pool.stats().total_inflight, 1);
3557
3558        task.abort();
3559        assert!(task.await.unwrap_err().is_cancelled());
3560        assert_eq!(pool.stats().total_inflight, 0);
3561
3562        tokio::time::timeout(Duration::from_secs(1), pool.close())
3563            .await
3564            .expect("an aborted in-command await must not prevent pool close");
3565    }
3566
3567    #[tokio::test]
3568    async fn pool_abort_after_alternate_slot_transfer_releases_reservation() {
3569        let alternate_started = Arc::new(tokio::sync::Notify::new());
3570        let pool = ConnectionPool::from_connections(
3571            vec![
3572                BlockingConn {
3573                    started: Arc::new(tokio::sync::Notify::new()),
3574                    release: Arc::new(tokio::sync::Notify::new()),
3575                },
3576                BlockingConn {
3577                    started: Arc::clone(&alternate_started),
3578                    release: Arc::new(tokio::sync::Notify::new()),
3579                },
3580            ],
3581            PoolConfig::default().dispatch(DispatchStrategy::RoundRobin),
3582        )
3583        .unwrap();
3584
3585        // The first round-robin reservation prefers slot 0. Holding its mutex
3586        // forces acquisition to transfer that reservation to idle slot 1.
3587        let preferred_guard = pool.inner.connections[0].lock().await;
3588        let task_pool = pool.clone();
3589        let task = tokio::spawn(async move { task_pool.execute(Ping::new()).await });
3590
3591        tokio::time::timeout(Duration::from_secs(1), alternate_started.notified())
3592            .await
3593            .expect("command should start on the alternate slot");
3594        assert_eq!(
3595            pool.inner.connections[0].inflight.load(Ordering::Acquire),
3596            0
3597        );
3598        assert_eq!(
3599            pool.inner.connections[1].inflight.load(Ordering::Acquire),
3600            1
3601        );
3602        assert_eq!(
3603            pool.inner.admission_state.load(Ordering::Acquire) & POOL_ACTIVE_MASK,
3604            1
3605        );
3606
3607        task.abort();
3608        assert!(task.await.unwrap_err().is_cancelled());
3609        assert_eq!(
3610            pool.inner.connections[0].inflight.load(Ordering::Acquire),
3611            0
3612        );
3613        assert_eq!(
3614            pool.inner.connections[1].inflight.load(Ordering::Acquire),
3615            0
3616        );
3617        assert_eq!(
3618            pool.inner.admission_state.load(Ordering::Acquire) & POOL_ACTIVE_MASK,
3619            0
3620        );
3621        drop(preferred_guard);
3622
3623        tokio::time::timeout(Duration::from_secs(1), pool.close())
3624            .await
3625            .expect("an aborted alternate-slot command must not prevent pool close");
3626    }
3627
3628    #[tokio::test]
3629    async fn pool_close_rejects_new_commands() {
3630        let conns = vec![MockConn::new(
3631            0,
3632            vec![Frame::SimpleString(Bytes::from("PONG"))],
3633        )];
3634        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
3635        assert!(!pool.is_closed());
3636
3637        let executor_view = pool.clone();
3638        pool.close().await;
3639
3640        assert!(executor_view.is_closed());
3641
3642        // The inherent execute path rejects new commands.
3643        let result: Result<Bytes, _> = executor_view.execute(Ping::new()).await;
3644        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
3645
3646        // The RedisExecutor path rejects them too.
3647        let mut executor_view = executor_view;
3648        let result: Result<Bytes, _> =
3649            RedisExecutor::execute(&mut executor_view, Ping::new()).await;
3650        assert!(matches!(result, Err(RedisError::ConnectionClosed)));
3651    }
3652
3653    #[tokio::test]
3654    async fn pool_close_is_prompt_when_idle() {
3655        let conns = vec![MockConn::new(
3656            0,
3657            vec![Frame::SimpleString(Bytes::from("PONG"))],
3658        )];
3659        let pool = ConnectionPool::from_connections(conns, PoolConfig::default()).unwrap();
3660        // An idle pool drains immediately; guard against a hang with a timeout.
3661        tokio::time::timeout(Duration::from_secs(1), pool.close())
3662            .await
3663            .expect("close on an idle pool should return promptly");
3664    }
3665
3666    #[tokio::test]
3667    async fn pool_close_drains_in_flight_command() {
3668        let started = Arc::new(tokio::sync::Notify::new());
3669        let release = Arc::new(tokio::sync::Notify::new());
3670        let conn = BlockingConn {
3671            started: Arc::clone(&started),
3672            release: Arc::clone(&release),
3673        };
3674        let pool = ConnectionPool::from_connections(vec![conn], PoolConfig::default()).unwrap();
3675
3676        // Start a command and wait until it is actually in-flight.
3677        let cmd_pool = pool.clone();
3678        let cmd_task = tokio::spawn(async move { cmd_pool.execute(Ping::new()).await });
3679        started.notified().await;
3680
3681        // close() must not return while the command is still in-flight.
3682        let close_task = tokio::spawn(async move { pool.close().await });
3683        tokio::time::sleep(Duration::from_millis(20)).await;
3684        assert!(
3685            !close_task.is_finished(),
3686            "close() should block until in-flight commands drain"
3687        );
3688
3689        // Release the command; both it and close() then complete.
3690        release.notify_one();
3691        let cmd_result: Result<Bytes, _> = cmd_task.await.unwrap();
3692        assert!(cmd_result.is_ok());
3693        tokio::time::timeout(Duration::from_secs(1), close_task)
3694            .await
3695            .expect("close() should return once the pool drains")
3696            .unwrap();
3697    }
3698
3699    #[tokio::test]
3700    async fn pool_close_drains_in_flight_active_probe() {
3701        let started = Arc::new(tokio::sync::Notify::new());
3702        let release = Arc::new(tokio::sync::Notify::new());
3703        let pool = ConnectionPool::from_connections(
3704            vec![BlockingConn {
3705                started: Arc::clone(&started),
3706                release: Arc::clone(&release),
3707            }],
3708            PoolConfig::default(),
3709        )
3710        .unwrap();
3711        let prober = pool.spawn_health_prober(
3712            HealthProberConfig::default()
3713                .interval(Duration::from_secs(60))
3714                .timeout(Duration::from_secs(60)),
3715        );
3716        started.notified().await;
3717
3718        let close_pool = pool.clone();
3719        let close_task = tokio::spawn(async move { close_pool.close().await });
3720        tokio::time::sleep(Duration::from_millis(20)).await;
3721        assert!(
3722            !close_task.is_finished(),
3723            "close() must wait while an admitted active probe owns a connection"
3724        );
3725
3726        release.notify_one();
3727        tokio::time::timeout(Duration::from_secs(1), close_task)
3728            .await
3729            .expect("close should finish after the active probe completes")
3730            .unwrap();
3731        prober.shutdown().await;
3732    }
3733
3734    #[tokio::test]
3735    async fn reservation_transfer_rejects_a_reaped_growth_slot() {
3736        let pool = ConnectionPool::from_connections(
3737            vec![MockConn::new(0, vec![]), MockConn::new(1, vec![])],
3738            PoolConfig::default(),
3739        )
3740        .unwrap();
3741        {
3742            let _topology = pool.inner.topology_write();
3743            pool.inner.connections[1]
3744                .active
3745                .store(false, Ordering::Release);
3746            *pool.inner.connections[1].connection.try_lock().unwrap() = None;
3747        }
3748
3749        let admission = PoolAdmission::try_new(&pool.inner.admission_state).unwrap();
3750        let mut reservation = InflightReservation::new(&pool.inner.connections[0], 0, admission);
3751        assert!(
3752            !pool
3753                .inner
3754                .transfer_reservation_if_active(&mut reservation, 1),
3755            "an inactive slot must be rejected after losing the reaper race"
3756        );
3757        assert_eq!(reservation.index(), 0);
3758        assert_eq!(
3759            pool.inner.connections[0].inflight.load(Ordering::Acquire),
3760            1
3761        );
3762        assert_eq!(
3763            pool.inner.connections[1].inflight.load(Ordering::Acquire),
3764            0
3765        );
3766    }
3767
3768    #[tokio::test]
3769    async fn lazy_pool_creates_nothing_until_first_command() {
3770        let factory = MockFactory::new(vec![MockConn::new(
3771            0,
3772            vec![Frame::SimpleString(Bytes::from("PONG"))],
3773        )]);
3774        let observed = factory.clone();
3775        let pool = ConnectionPool::connect_lazy(PoolConfig::default().max_size(2), factory);
3776
3777        assert_eq!(pool.size(), 0);
3778        assert_eq!(observed.creates(), 0);
3779
3780        let pong: Bytes = pool.execute(Ping::new()).await.unwrap();
3781        assert_eq!(pong, "PONG");
3782        assert_eq!(pool.size(), 1);
3783        assert_eq!(observed.creates(), 1);
3784        assert_eq!(pool.stats().min_size, 0);
3785        assert_eq!(pool.stats().max_size, 2);
3786    }
3787
3788    #[tokio::test]
3789    async fn lazy_pool_growth_obeys_acquisition_timeout() {
3790        let pool = ConnectionPool::connect_lazy(
3791            PoolConfig::default()
3792                .max_size(1)
3793                .acquisition_timeout(Duration::from_millis(10)),
3794            SlowFactory::new(Duration::from_secs(60)),
3795        );
3796
3797        let result: Result<Bytes, _> =
3798            tokio::time::timeout(Duration::from_secs(1), pool.execute(Ping::new()))
3799                .await
3800                .expect("pool acquisition timeout must cancel a slow factory");
3801        assert!(matches!(
3802            result,
3803            Err(RedisError::PoolAcquisitionTimeout { .. })
3804        ));
3805        assert_eq!(pool.size(), 0);
3806    }
3807
3808    #[tokio::test]
3809    async fn contended_growth_obeys_command_deadline() {
3810        let pool = ConnectionPool::connect_with_factory(
3811            PoolConfig::default()
3812                .bounds(1, 2)
3813                .disable_acquisition_timeout(),
3814            SlowFactory::new(Duration::from_millis(50)),
3815        )
3816        .await
3817        .unwrap();
3818        let busy = pool.inner.connections[0].lock().await;
3819
3820        let result: Result<Bytes, _> = pool
3821            .execute(WithDeadline::after(Ping::new(), Duration::from_millis(10)))
3822            .await;
3823        assert!(matches!(result, Err(RedisError::CommandTimeout)));
3824        assert_eq!(pool.size(), 1, "timed-out growth must not activate a slot");
3825        drop(busy);
3826    }
3827
3828    #[tokio::test]
3829    async fn cancelling_lazy_growth_releases_the_scale_lock() {
3830        let calls = Arc::new(AtomicUsize::new(0));
3831        let first_entered = Arc::new(tokio::sync::Notify::new());
3832        let pool = ConnectionPool::connect_lazy(
3833            PoolConfig::default().max_size(1),
3834            CancelThenConnectFactory {
3835                calls: Arc::clone(&calls),
3836                first_entered: Arc::clone(&first_entered),
3837            },
3838        );
3839
3840        let first_pool = pool.clone();
3841        let first = tokio::spawn(async move { first_pool.execute(Ping::new()).await });
3842        first_entered.notified().await;
3843        first.abort();
3844        assert!(first.await.unwrap_err().is_cancelled());
3845
3846        let pong: Bytes = tokio::time::timeout(Duration::from_secs(1), pool.execute(Ping::new()))
3847            .await
3848            .expect("a canceled factory must release the pool's scale lock")
3849            .unwrap();
3850        assert_eq!(pong, "PONG");
3851        assert_eq!(calls.load(Ordering::Acquire), 2);
3852    }
3853
3854    #[tokio::test]
3855    async fn dynamic_pool_grows_on_contention_up_to_max_size() {
3856        let factory = DelayedFactory::new(Duration::from_millis(30));
3857        let observed = factory.clone();
3858        let pool =
3859            ConnectionPool::connect_with_factory(PoolConfig::default().bounds(1, 3), factory)
3860                .await
3861                .unwrap();
3862
3863        let mut tasks = Vec::new();
3864        for _ in 0..3 {
3865            let pool = pool.clone();
3866            tasks.push(tokio::spawn(async move { pool.execute(Ping::new()).await }));
3867        }
3868        for task in tasks {
3869            let result: Result<Bytes, _> = task.await.unwrap();
3870            assert_eq!(result.unwrap(), "PONG");
3871        }
3872
3873        assert_eq!(pool.size(), 3);
3874        assert_eq!(observed.creates(), 3);
3875        let stats = pool.stats();
3876        assert_eq!((stats.min_size, stats.max_size), (1, 3));
3877        assert_eq!(stats.idle_count, 3);
3878    }
3879
3880    #[tokio::test]
3881    async fn idle_reaping_shrinks_to_minimum_and_updates_stats_and_metrics() {
3882        let recorder = Arc::new(CountingRecorder::default());
3883        let factory = DelayedFactory::new(Duration::from_millis(20));
3884        let pool = ConnectionPool::connect_with_factory(
3885            PoolConfig::default()
3886                .bounds(1, 3)
3887                .idle_timeout(Duration::from_millis(1))
3888                .metrics_recorder(recorder.clone()),
3889            factory,
3890        )
3891        .await
3892        .unwrap();
3893
3894        let mut tasks = Vec::new();
3895        for _ in 0..3 {
3896            let pool = pool.clone();
3897            tasks.push(tokio::spawn(async move { pool.execute(Ping::new()).await }));
3898        }
3899        for task in tasks {
3900            let result: Result<Bytes, _> = task.await.unwrap();
3901            result.unwrap();
3902        }
3903        assert_eq!(pool.size(), 3);
3904
3905        tokio::time::sleep(Duration::from_millis(5)).await;
3906        assert_eq!(pool.reap_idle_connections().await, 2);
3907        let stats = pool.stats();
3908        assert_eq!(stats.size, 1);
3909        assert_eq!(stats.reaped_connections, 2);
3910        assert_eq!(recorder.reaped_connections.load(Ordering::Acquire), 2);
3911    }
3912
3913    #[tokio::test]
3914    async fn active_ping_prober_updates_stats_metrics_and_stops_on_shutdown() {
3915        let recorder = Arc::new(CountingRecorder::default());
3916        let pool = ConnectionPool::from_connections(
3917            vec![MockConn::new(
3918                0,
3919                vec![Frame::SimpleString(Bytes::from("PONG"))],
3920            )],
3921            PoolConfig::default().metrics_recorder(recorder.clone()),
3922        )
3923        .unwrap();
3924        assert_eq!(pool.stats().unknown_health_count, 1);
3925
3926        let handle = pool
3927            .spawn_health_prober(HealthProberConfig::default().interval(Duration::from_secs(60)));
3928        tokio::time::timeout(Duration::from_secs(1), async {
3929            while pool.stats().healthy_count != 1 {
3930                tokio::task::yield_now().await;
3931            }
3932        })
3933        .await
3934        .expect("the immediate prober sweep should complete");
3935
3936        handle.shutdown().await;
3937        let probes = recorder.health_probes.lock().unwrap().clone();
3938        assert_eq!(probes, vec![(HealthProbeKind::Ping, true, None)]);
3939        assert_eq!(pool.stats().unhealthy_count, 0);
3940    }
3941
3942    #[tokio::test]
3943    async fn active_prober_skips_busy_slots_without_marking_them_unhealthy() {
3944        let recorder = Arc::new(CountingRecorder::default());
3945        let pool = ConnectionPool::from_connections(
3946            vec![MockConn::new(
3947                0,
3948                vec![Frame::SimpleString(Bytes::from("PONG"))],
3949            )],
3950            PoolConfig::default().metrics_recorder(recorder.clone()),
3951        )
3952        .unwrap();
3953        let busy = pool.inner.connections[0].lock().await;
3954        let handle = pool.spawn_health_prober(
3955            HealthProberConfig::default()
3956                .interval(Duration::from_secs(60))
3957                .timeout(Duration::from_millis(5)),
3958        );
3959
3960        tokio::time::sleep(Duration::from_millis(20)).await;
3961        handle.shutdown().await;
3962        assert_eq!(pool.stats().unknown_health_count, 1);
3963        assert_eq!(pool.stats().unhealthy_count, 0);
3964        assert!(recorder.health_probes.lock().unwrap().is_empty());
3965        assert_eq!(recorder.health_check_failures.load(Ordering::Acquire), 0);
3966        drop(busy);
3967    }
3968
3969    #[test]
3970    fn role_probe_parser_accepts_primary_replica_and_sentinel() {
3971        let role = |name: &'static [u8]| {
3972            Frame::Array(Some(vec![Frame::BulkString(Some(Bytes::from_static(
3973                name,
3974            )))]))
3975        };
3976        assert_eq!(
3977            parse_role_frame(&role(b"master")).unwrap(),
3978            RedisRole::Primary
3979        );
3980        assert_eq!(
3981            parse_role_frame(&role(b"replica")).unwrap(),
3982            RedisRole::Replica
3983        );
3984        assert_eq!(
3985            parse_role_frame(&role(b"sentinel")).unwrap(),
3986            RedisRole::Sentinel
3987        );
3988    }
3989
3990    #[test]
3991    fn replication_probe_parser_checks_link_sync_and_offset_lag() {
3992        let healthy = parse_replication_info(
3993            "role:master\r\nconnected_slaves:2\r\nmaster_repl_offset:120\r\nslave0:ip=127.0.0.1,port=6380,state=online,offset=100,lag=0\r\nslave1:ip=127.0.0.1,port=6381,state=online,offset=115,lag=0\r\n",
3994        )
3995        .unwrap();
3996        assert!(healthy.link_up);
3997        assert!(!healthy.sync_in_progress);
3998        assert_eq!(healthy.lag_bytes, 20);
3999
4000        let no_replicas = parse_replication_info(
4001            "role:master\r\nconnected_slaves:0\r\nmaster_repl_offset:120\r\n",
4002        )
4003        .unwrap();
4004        assert!(!no_replicas.link_up);
4005        assert_eq!(no_replicas.lag_bytes, 0);
4006
4007        assert!(
4008            parse_replication_info(
4009                "role:slave\r\nmaster_link_status:up\r\nmaster_repl_offset:120\r\nslave_repl_offset:120\r\n",
4010            )
4011            .is_err(),
4012            "replica-local offsets cannot establish upstream byte lag"
4013        );
4014    }
4015
4016    #[tokio::test]
4017    async fn built_in_role_probe_enforces_expected_role() {
4018        let mut primary = MockConn::new(
4019            0,
4020            vec![Frame::Array(Some(vec![Frame::BulkString(Some(
4021                Bytes::from("master"),
4022            ))]))],
4023        );
4024        let result = RoleHealthProbe::new(RedisRole::Primary)
4025            .probe(&mut primary)
4026            .await
4027            .unwrap();
4028        assert!(result.healthy);
4029
4030        let mut replica = MockConn::new(
4031            1,
4032            vec![Frame::Array(Some(vec![Frame::BulkString(Some(
4033                Bytes::from("replica"),
4034            ))]))],
4035        );
4036        let result = RoleHealthProbe::new(RedisRole::Primary)
4037            .probe(&mut replica)
4038            .await
4039            .unwrap();
4040        assert!(!result.healthy);
4041    }
4042
4043    #[tokio::test]
4044    async fn built_in_replication_lag_probe_enforces_threshold() {
4045        let info = "role:master\r\nconnected_slaves:1\r\nmaster_repl_offset:120\r\nslave0:ip=127.0.0.1,port=6380,state=online,offset=100,lag=0\r\n";
4046        let mut connection = MockConn::new(0, vec![Frame::BulkString(Some(Bytes::from(info)))]);
4047        let result = ReplicationLagHealthProbe::new(19)
4048            .probe(&mut connection)
4049            .await
4050            .unwrap();
4051        assert!(!result.healthy);
4052        assert_eq!(result.replication_lag_bytes, Some(20));
4053    }
4054}