use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::Mutex as StdMutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use futures::Stream;
use redis_tower_core::{Command, ConnectionConfig, ProtocolVersion, RedisConnection, RedisError};
use tokio::sync::broadcast;
use tokio_stream::wrappers::BroadcastStream;
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
pub const DEFAULT_CONNECTION_EVENT_CAPACITY: usize = 64;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ConnectionDisconnectReason {
ConnectionError {
error: Arc<str>,
},
CommandTimeout,
ReadOnly,
MaintenanceHandoff {
sequence: u64,
},
Shutdown,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum ConnectionEvent {
Connected,
MaintenanceNotification {
kind: crate::maintenance::MaintenanceNotificationKind,
sequence: u64,
ttl: Duration,
},
ConnectFailed {
error: Arc<str>,
},
Disconnected {
reason: ConnectionDisconnectReason,
},
ReconnectAttempt {
attempt: usize,
delay: Duration,
},
ReconnectFailed {
attempt: usize,
error: Arc<str>,
},
Reconnected {
attempts: usize,
elapsed: Duration,
},
ReconnectExhausted {
attempts: usize,
},
Failover {
previous: Option<Arc<str>>,
current: Option<Arc<str>>,
},
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum ConnectionEventRecvError {
#[error("connection event subscriber lagged and skipped {skipped} event(s)")]
Lagged {
skipped: u64,
},
#[error("connection event stream closed")]
Closed,
}
#[derive(Debug, Clone)]
pub struct ConnectionEventBus {
tx: broadcast::Sender<ConnectionEvent>,
}
impl ConnectionEventBus {
pub fn new(capacity: usize) -> Self {
assert!(capacity > 0, "connection event capacity must be non-zero");
let (tx, _) = broadcast::channel(capacity);
Self { tx }
}
pub fn subscribe(&self) -> ConnectionEventStream {
ConnectionEventStream {
inner: BroadcastStream::new(self.tx.subscribe()),
}
}
pub fn subscriber_count(&self) -> usize {
self.tx.receiver_count()
}
pub fn publish(&self, event: ConnectionEvent) -> bool {
if self.tx.receiver_count() == 0 {
return false;
}
self.tx.send(event).is_ok()
}
pub fn publish_with(&self, make_event: impl FnOnce() -> ConnectionEvent) -> bool {
if self.tx.receiver_count() == 0 {
return false;
}
self.tx.send(make_event()).is_ok()
}
}
impl Default for ConnectionEventBus {
fn default() -> Self {
Self::new(DEFAULT_CONNECTION_EVENT_CAPACITY)
}
}
#[derive(Debug)]
pub struct ConnectionEventStream {
inner: BroadcastStream<ConnectionEvent>,
}
impl ConnectionEventStream {
pub async fn recv(&mut self) -> Result<ConnectionEvent, ConnectionEventRecvError> {
futures::future::poll_fn(|cx| Pin::new(&mut *self).poll_next(cx))
.await
.unwrap_or(Err(ConnectionEventRecvError::Closed))
}
}
impl Stream for ConnectionEventStream {
type Item = Result<ConnectionEvent, ConnectionEventRecvError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
match Pin::new(&mut self.inner).poll_next(cx) {
Poll::Ready(Some(Ok(event))) => Poll::Ready(Some(Ok(event))),
Poll::Ready(Some(Err(BroadcastStreamRecvError::Lagged(skipped)))) => {
Poll::Ready(Some(Err(ConnectionEventRecvError::Lagged { skipped })))
}
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
}
pub trait ConnectionFactory: Send + Sync + 'static {
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>>;
}
impl<F, Fut> ConnectionFactory for F
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<RedisConnection, RedisError>> + Send + 'static,
{
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>> {
Box::pin((self)())
}
}
impl ConnectionFactory for Arc<dyn ConnectionFactory> {
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>> {
self.as_ref().connect()
}
}
pub struct UrlConnectionFactory {
url: String,
connection_config: ConnectionConfig,
#[cfg(any(feature = "tls-rustls", feature = "tls-native-tls"))]
tls: Option<std::sync::Arc<redis_tower_core::tls::TlsConfig>>,
}
impl UrlConnectionFactory {
pub fn new(url: impl Into<String>) -> Self {
Self {
url: url.into(),
connection_config: ConnectionConfig::default(),
#[cfg(any(feature = "tls-rustls", feature = "tls-native-tls"))]
tls: None,
}
}
pub fn with_connection_config(mut self, config: ConnectionConfig) -> Self {
self.connection_config = config;
self
}
#[cfg(any(feature = "tls-rustls", feature = "tls-native-tls"))]
pub fn with_tls(mut self, tls: redis_tower_core::tls::TlsConfig) -> Self {
self.tls = Some(std::sync::Arc::new(tls));
self
}
}
impl ConnectionFactory for UrlConnectionFactory {
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>> {
let url = self.url.clone();
let connection_config = self.connection_config.clone();
#[cfg(any(feature = "tls-rustls", feature = "tls-native-tls"))]
if let Some(tls) = self.tls.clone() {
return Box::pin(async move {
RedisConnection::connect_url_with_tls_and_config(&url, &tls, &connection_config)
.await
});
}
Box::pin(
async move { RedisConnection::connect_url_with_config(&url, &connection_config).await },
)
}
}
impl crate::pool::PoolFactory for UrlConnectionFactory {
type Connection = RedisConnection;
fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
ConnectionFactory::connect(self)
}
}
pub struct AddrConnectionFactory {
addr: String,
connection_config: ConnectionConfig,
}
impl AddrConnectionFactory {
pub fn new(addr: impl Into<String>) -> Self {
Self {
addr: addr.into(),
connection_config: ConnectionConfig::default(),
}
}
pub fn with_connection_config(mut self, config: ConnectionConfig) -> Self {
self.connection_config = config;
self
}
}
impl ConnectionFactory for AddrConnectionFactory {
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>> {
let addr = self.addr.clone();
let connection_config = self.connection_config.clone();
Box::pin(
async move { RedisConnection::connect_with_config(&addr, &connection_config).await },
)
}
}
impl crate::pool::PoolFactory for AddrConnectionFactory {
type Connection = RedisConnection;
fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
ConnectionFactory::connect(self)
}
}
pub struct Resp3AddrConnectionFactory {
addr: String,
connection_config: ConnectionConfig,
}
impl Resp3AddrConnectionFactory {
pub fn new(addr: impl Into<String>) -> Self {
Self {
addr: addr.into(),
connection_config: ConnectionConfig::default().with_protocol(ProtocolVersion::Resp3),
}
}
pub fn with_connection_config(mut self, config: ConnectionConfig) -> Self {
self.connection_config = config.with_protocol(ProtocolVersion::Resp3);
self
}
}
impl ConnectionFactory for Resp3AddrConnectionFactory {
fn connect(&self) -> Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>> {
let addr = self.addr.clone();
let connection_config = self.connection_config.clone();
Box::pin(
async move { RedisConnection::connect_with_config(&addr, &connection_config).await },
)
}
}
impl crate::pool::PoolFactory for Resp3AddrConnectionFactory {
type Connection = RedisConnection;
fn create(&self) -> Pin<Box<dyn Future<Output = Result<Self::Connection, RedisError>> + Send>> {
ConnectionFactory::connect(self)
}
}
#[derive(Debug, Clone)]
pub struct ReconnectConfig {
pub max_retries: Option<usize>,
pub base_delay: Duration,
pub max_delay: Duration,
pub jitter: bool,
pub connect_timeout: Option<Duration>,
}
impl Default for ReconnectConfig {
fn default() -> Self {
Self {
max_retries: None,
base_delay: Duration::from_millis(100),
max_delay: Duration::from_secs(5),
jitter: true,
connect_timeout: None,
}
}
}
impl ReconnectConfig {
#[must_use]
pub fn max_retries(mut self, n: usize) -> Self {
self.max_retries = Some(n);
self
}
#[must_use]
pub fn base_delay(mut self, d: Duration) -> Self {
self.base_delay = d;
self
}
#[must_use]
pub fn max_delay(mut self, d: Duration) -> Self {
self.max_delay = d;
self
}
#[must_use]
pub fn jitter(mut self, enabled: bool) -> Self {
self.jitter = enabled;
self
}
#[must_use]
pub fn connect_timeout(mut self, d: Duration) -> Self {
self.connect_timeout = Some(d);
self
}
pub(crate) fn attempt_exhausted(&self, attempt: usize) -> bool {
self.max_retries.map(|max| attempt > max).unwrap_or(false)
}
pub(crate) fn total_attempt_budget(&self) -> Option<usize> {
self.max_retries.map(|max| max.saturating_add(1))
}
pub(crate) fn delay_for_attempt(&self, attempt: usize) -> Duration {
let cap = self
.base_delay
.saturating_mul(1u32.wrapping_shl(attempt.min(31) as u32))
.min(self.max_delay);
if self.jitter {
let nanos = cap.as_nanos() as u64;
if nanos == 0 {
return Duration::ZERO;
}
Duration::from_nanos(rand::random::<u64>() % nanos)
} else {
cap
}
}
}
pub(crate) async fn connect_with_timeout(
factory: &dyn ConnectionFactory,
timeout: Option<Duration>,
) -> Result<RedisConnection, RedisError> {
match timeout {
Some(timeout) => tokio::time::timeout(timeout, factory.connect())
.await
.map_err(|_| RedisError::ConnectTimeout)?,
None => factory.connect().await,
}
}
type ReconnectFuture = Pin<Box<dyn Future<Output = Result<RedisConnection, RedisError>> + Send>>;
pub(crate) enum ConnState {
Connected(RedisConnection),
WaitingToReconnect {
attempt: usize,
sleep: Pin<Box<tokio::time::Sleep>>,
started: Instant,
},
Reconnecting {
attempt: usize,
future: ReconnectFuture,
started: Instant,
},
Failed,
}
pub struct ResilientConnection {
pub(crate) factory: Arc<dyn ConnectionFactory>,
pub(crate) config: ReconnectConfig,
pub(crate) state: ConnState,
pub(crate) needs_reconnect: Arc<AtomicBool>,
event_bus: Option<ConnectionEventBus>,
disconnect_reported: Option<Arc<AtomicBool>>,
shutdown_reported: Option<Arc<AtomicBool>>,
lifecycle_lock: Option<Arc<StdMutex<()>>>,
pub(crate) on_connect: Option<Arc<dyn Fn() + Send + Sync>>,
pub(crate) on_reconnect: Option<Arc<dyn Fn(usize) + Send + Sync>>,
}
impl ResilientConnection {
pub async fn new(
factory: impl ConnectionFactory,
config: ReconnectConfig,
) -> Result<Self, RedisError> {
Self::new_inner(factory, config, None).await
}
pub async fn new_with_events(
factory: impl ConnectionFactory,
config: ReconnectConfig,
events: ConnectionEventBus,
) -> Result<Self, RedisError> {
Self::new_inner(factory, config, Some(events)).await
}
async fn new_inner(
factory: impl ConnectionFactory,
config: ReconnectConfig,
event_bus: Option<ConnectionEventBus>,
) -> Result<Self, RedisError> {
let factory = Arc::new(factory);
let result = connect_with_timeout(factory.as_ref(), config.connect_timeout).await;
let conn = match result {
Ok(conn) => conn,
Err(error) => {
if let Some(events) = &event_bus {
events.publish_with(|| ConnectionEvent::ConnectFailed {
error: Arc::from(error.to_string()),
});
}
return Err(error);
}
};
if let Some(events) = &event_bus {
events.publish(ConnectionEvent::Connected);
}
let disconnect_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
let shutdown_reported = event_bus.as_ref().map(|_| Arc::new(AtomicBool::new(false)));
let lifecycle_lock = event_bus.as_ref().map(|_| Arc::new(StdMutex::new(())));
Ok(Self {
factory,
config,
state: ConnState::Connected(conn),
needs_reconnect: Arc::new(AtomicBool::new(false)),
event_bus,
disconnect_reported,
shutdown_reported,
lifecycle_lock,
on_connect: None,
on_reconnect: None,
})
}
pub fn on_connect(mut self, f: impl Fn() + Send + Sync + 'static) -> Self {
self.on_connect = Some(Arc::new(f));
self
}
pub fn on_reconnect(mut self, f: impl Fn(usize) + Send + Sync + 'static) -> Self {
self.on_reconnect = Some(Arc::new(f));
self
}
pub async fn execute<Cmd: Command>(&mut self, cmd: Cmd) -> Result<Cmd::Response, RedisError> {
match &mut self.state {
ConnState::Connected(conn) => {
let result = conn.execute(cmd).await;
if let Err(ref e) = result
&& e.is_connection_error()
{
self.needs_reconnect.store(true, Ordering::Release);
publish_disconnect_before_shutdown(
self.event_bus.as_ref(),
self.disconnect_reported.as_deref(),
self.shutdown_reported.as_deref(),
self.lifecycle_lock.as_deref(),
e,
);
}
result
}
_ => Err(RedisError::ConnectionClosed),
}
}
fn trigger_reconnect(&mut self, attempt: usize, started: Instant) {
if self.config.attempt_exhausted(attempt) {
if let Some(events) = &self.event_bus {
events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
}
self.state = ConnState::Failed;
return;
}
let delay = self.config.delay_for_attempt(attempt);
tracing::warn!(attempt, delay = ?delay, "redis: connection lost, reconnecting");
if let Some(events) = &self.event_bus {
events.publish(ConnectionEvent::ReconnectAttempt {
attempt: attempt + 1,
delay,
});
}
self.state = ConnState::WaitingToReconnect {
attempt,
sleep: Box::pin(tokio::time::sleep(delay)),
started,
};
}
}
impl Drop for ResilientConnection {
fn drop(&mut self) {
let _guard = self.lifecycle_lock.as_ref().map(|lock| {
lock.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
});
publish_shutdown_once(self.event_bus.as_ref(), self.shutdown_reported.as_deref());
}
}
impl<Cmd: Command> tower_service::Service<Cmd> for ResilientConnection {
type Response = Cmd::Response;
type Error = RedisError;
type Future = Pin<Box<dyn Future<Output = Result<Cmd::Response, RedisError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
if self.needs_reconnect.swap(false, Ordering::Acquire)
&& matches!(self.state, ConnState::Connected(_))
{
self.trigger_reconnect(0, Instant::now());
}
loop {
match &mut self.state {
ConnState::Connected(_) => return Poll::Ready(Ok(())),
ConnState::Failed => {
return Poll::Ready(Err(RedisError::ReconnectFailed {
attempts: self.config.total_attempt_budget().unwrap_or(0),
last_error: Arc::new(RedisError::ConnectionClosed),
}));
}
ConnState::WaitingToReconnect {
attempt,
sleep,
started,
} => match sleep.as_mut().poll(cx) {
Poll::Ready(()) => {
let attempt = *attempt;
let started = *started;
let connect_timeout = self.config.connect_timeout;
let factory = Arc::clone(&self.factory);
let future: ReconnectFuture = Box::pin(async move {
connect_with_timeout(factory.as_ref(), connect_timeout).await
});
self.state = ConnState::Reconnecting {
attempt,
future,
started,
};
}
Poll::Pending => return Poll::Pending,
},
ConnState::Reconnecting {
attempt,
future,
started,
} => match future.as_mut().poll(cx) {
Poll::Ready(Ok(conn)) => {
let attempt = *attempt;
let elapsed = started.elapsed();
let elapsed_ms = elapsed.as_millis();
self.state = ConnState::Connected(conn);
if let Some(disconnect_reported) = &self.disconnect_reported {
disconnect_reported.store(false, Ordering::Release);
}
tracing::info!(attempt, elapsed_ms, "redis: reconnected successfully");
if let Some(events) = &self.event_bus {
events.publish(ConnectionEvent::Reconnected {
attempts: attempt + 1,
elapsed,
});
}
if attempt > 0
&& let Some(ref cb) = self.on_reconnect
{
cb(attempt);
}
if let Some(ref cb) = self.on_connect {
cb();
}
return Poll::Ready(Ok(()));
}
Poll::Ready(Err(e)) => {
let attempt = *attempt;
let started = *started;
tracing::warn!(attempt, error = %e, "redis: reconnect attempt failed");
if let Some(events) = &self.event_bus {
events.publish_with(|| ConnectionEvent::ReconnectFailed {
attempt: attempt + 1,
error: Arc::from(e.to_string()),
});
}
self.trigger_reconnect(attempt + 1, started);
}
Poll::Pending => return Poll::Pending,
},
}
}
}
fn call(&mut self, cmd: Cmd) -> Self::Future {
let conn = match &mut self.state {
ConnState::Connected(conn) => conn,
_ => return Box::pin(async { Err(RedisError::ConnectionClosed) }),
};
let future = <RedisConnection as tower_service::Service<Cmd>>::call(conn, cmd);
let needs_reconnect = Arc::clone(&self.needs_reconnect);
let event_bus = self.event_bus.clone();
let disconnect_reported = self.disconnect_reported.clone();
let shutdown_reported = self.shutdown_reported.clone();
let lifecycle_lock = self.lifecycle_lock.clone();
Box::pin(async move {
let result = future.await;
if let Err(ref e) = result
&& e.is_connection_error()
{
needs_reconnect.store(true, Ordering::Release);
publish_disconnect_before_shutdown(
event_bus.as_ref(),
disconnect_reported.as_deref(),
shutdown_reported.as_deref(),
lifecycle_lock.as_deref(),
e,
);
}
result
})
}
}
pub(crate) fn publish_disconnect_once(
event_bus: Option<&ConnectionEventBus>,
disconnect_reported: Option<&AtomicBool>,
error: &RedisError,
) {
let (Some(events), Some(disconnect_reported)) = (event_bus, disconnect_reported) else {
return;
};
if disconnect_reported
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return;
}
events.publish_with(|| ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError {
error: Arc::from(error.to_string()),
},
});
}
pub(crate) fn publish_disconnect_before_shutdown(
event_bus: Option<&ConnectionEventBus>,
disconnect_reported: Option<&AtomicBool>,
shutdown_reported: Option<&AtomicBool>,
lifecycle_lock: Option<&StdMutex<()>>,
error: &RedisError,
) {
let _guard = lifecycle_lock.map(|lock| {
lock.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
});
if shutdown_reported.is_some_and(|reported| reported.load(Ordering::Acquire)) {
return;
}
publish_disconnect_once(event_bus, disconnect_reported, error);
}
pub(crate) fn publish_shutdown_once(
event_bus: Option<&ConnectionEventBus>,
shutdown_reported: Option<&AtomicBool>,
) {
let (Some(events), Some(shutdown_reported)) = (event_bus, shutdown_reported) else {
return;
};
if shutdown_reported
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return;
}
events.publish(ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
});
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn event_bus_delivers_ordered_events_to_multiple_subscribers() {
let events = ConnectionEventBus::new(8);
let mut first = events.subscribe();
let mut second = events.subscribe();
let expected = [
ConnectionEvent::Connected,
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::from_millis(25),
},
ConnectionEvent::Reconnected {
attempts: 1,
elapsed: Duration::from_millis(30),
},
ConnectionEvent::Failover {
previous: Some(Arc::from("redis-a:6379")),
current: Some(Arc::from("redis-b:6379")),
},
];
for event in expected.iter().cloned() {
assert!(events.publish(event));
}
for expected_event in &expected {
assert_eq!(first.recv().await.unwrap(), expected_event.clone());
assert_eq!(second.recv().await.unwrap(), expected_event.clone());
}
}
#[tokio::test]
async fn event_stream_reports_lag_then_continues_with_retained_event() {
let events = ConnectionEventBus::new(1);
let mut stream = events.subscribe();
events.publish(ConnectionEvent::Connected);
events.publish(ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::ZERO,
});
events.publish(ConnectionEvent::ReconnectExhausted { attempts: 1 });
let skipped = match stream.recv().await.unwrap_err() {
ConnectionEventRecvError::Lagged { skipped } => skipped,
other => panic!("expected lag error, got {other:?}"),
};
assert!(skipped >= 1);
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectExhausted { attempts: 1 }
);
}
#[tokio::test]
async fn event_stream_reports_closed_after_last_publisher_drops() {
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
drop(events);
assert_eq!(
stream.recv().await.unwrap_err(),
ConnectionEventRecvError::Closed
);
}
#[test]
fn lazy_publish_skips_payload_work_without_subscribers() {
let events = ConnectionEventBus::new(4);
let constructed = AtomicBool::new(false);
assert!(!events.publish_with(|| {
constructed.store(true, Ordering::Release);
ConnectionEvent::Connected
}));
assert!(!constructed.load(Ordering::Acquire));
}
#[tokio::test]
async fn shutdown_is_distinct_from_an_outage_and_emitted_once() {
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let disconnect_reported = AtomicBool::new(false);
let shutdown_reported = AtomicBool::new(false);
publish_disconnect_once(
Some(&events),
Some(&disconnect_reported),
&RedisError::ConnectionClosed,
);
publish_shutdown_once(Some(&events), Some(&shutdown_reported));
publish_shutdown_once(Some(&events), Some(&shutdown_reported));
assert!(matches!(
stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
assert!(
tokio::time::timeout(Duration::from_millis(10), stream.recv())
.await
.is_err()
);
}
#[tokio::test]
async fn initial_connect_failure_is_observable() {
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let factory = || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) };
let result = ResilientConnection::new_with_events(
factory,
ReconnectConfig::default(),
events.clone(),
)
.await;
assert!(result.is_err());
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ConnectFailed {
error: Arc::from("connection closed"),
}
);
}
#[tokio::test]
async fn initial_connect_timeout_is_observable() {
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let factory =
|| async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
let result = ResilientConnection::new_with_events(
factory,
ReconnectConfig::default().connect_timeout(Duration::from_millis(10)),
events,
)
.await;
assert!(matches!(result, Err(RedisError::ConnectTimeout)));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ConnectFailed {
error: Arc::from(RedisError::ConnectTimeout.to_string()),
}
);
}
#[tokio::test]
async fn dropping_healthy_connection_publishes_shutdown_once() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let _stream = stream;
futures::future::pending::<()>().await;
});
let factory = move || async move {
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|error| RedisError::connection(addr.to_string(), error))?;
Ok(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
};
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let connection = ResilientConnection::new_with_events(
factory,
ReconnectConfig::default(),
events.clone(),
)
.await
.unwrap();
assert_eq!(stream.recv().await.unwrap(), ConnectionEvent::Connected);
drop(connection);
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
assert!(
tokio::time::timeout(Duration::from_millis(10), stream.recv())
.await
.is_err()
);
server.abort();
}
#[test]
fn named_factories_retain_connection_config() {
let limits = redis_tower_core::RespLimits {
max_frame_size: 4096,
max_depth: 7,
};
let config = ConnectionConfig::new()
.with_protocol(ProtocolVersion::Resp2)
.with_resp_limits(limits);
let addr =
AddrConnectionFactory::new("127.0.0.1:6379").with_connection_config(config.clone());
assert_eq!(addr.connection_config.protocol(), ProtocolVersion::Resp2);
assert_eq!(addr.connection_config.resp_limits(), limits);
let url = UrlConnectionFactory::new("redis://127.0.0.1:6379")
.with_connection_config(config.clone());
assert_eq!(url.connection_config.protocol(), ProtocolVersion::Resp2);
assert_eq!(url.connection_config.resp_limits(), limits);
let resp3 =
Resp3AddrConnectionFactory::new("127.0.0.1:6379").with_connection_config(config);
assert_eq!(resp3.connection_config.protocol(), ProtocolVersion::Resp3);
assert_eq!(resp3.connection_config.resp_limits(), limits);
}
#[test]
fn named_factories_can_replace_pooled_connections() {
fn assert_pool_factory<F: crate::pool::PoolFactory<Connection = RedisConnection>>() {}
assert_pool_factory::<AddrConnectionFactory>();
assert_pool_factory::<UrlConnectionFactory>();
assert_pool_factory::<Resp3AddrConnectionFactory>();
}
#[test]
fn jitter_produces_different_delays() {
let config = ReconnectConfig::default(); let delays: Vec<Duration> = (0..100).map(|_| config.delay_for_attempt(0)).collect();
let first = delays[0];
assert!(
delays.iter().any(|d| *d != first),
"all 100 jittered delays were identical — jitter may not be working"
);
}
#[test]
fn jitter_delays_are_within_cap() {
let config = ReconnectConfig::default(); let cap = Duration::from_millis(100); for _ in 0..1000 {
let d = config.delay_for_attempt(0);
assert!(d < cap, "jittered delay {d:?} exceeded cap {cap:?}");
}
}
#[test]
fn no_jitter_produces_deterministic_delays() {
let config = ReconnectConfig::default().jitter(false);
let d0 = config.delay_for_attempt(0);
let d0b = config.delay_for_attempt(0);
assert_eq!(d0, d0b, "delays should be identical with jitter disabled");
assert_eq!(
d0,
Duration::from_millis(100),
"attempt 0 without jitter should equal base_delay"
);
}
#[test]
fn no_jitter_exponential_backoff() {
let config = ReconnectConfig::default().jitter(false);
assert_eq!(config.delay_for_attempt(0), Duration::from_millis(100));
assert_eq!(config.delay_for_attempt(1), Duration::from_millis(200));
assert_eq!(config.delay_for_attempt(2), Duration::from_millis(400));
assert_eq!(config.delay_for_attempt(3), Duration::from_millis(800));
}
#[test]
fn no_jitter_capped_at_max_delay() {
let config = ReconnectConfig::default().jitter(false);
assert_eq!(config.delay_for_attempt(6), Duration::from_secs(5));
}
#[test]
fn zero_cap_returns_zero() {
let config = ReconnectConfig {
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: true,
..Default::default()
};
assert_eq!(config.delay_for_attempt(0), Duration::ZERO);
}
#[test]
fn reconnect_config_connect_timeout() {
let cfg = ReconnectConfig::default().connect_timeout(Duration::from_secs(2));
assert_eq!(cfg.connect_timeout, Some(Duration::from_secs(2)));
}
#[test]
fn reconnect_config_connect_timeout_default_is_none() {
let cfg = ReconnectConfig::default();
assert_eq!(cfg.connect_timeout, None);
}
#[test]
fn max_retries_zero_allows_one_attempt() {
let config = ReconnectConfig::default().max_retries(0);
assert!(!config.attempt_exhausted(0));
assert!(config.attempt_exhausted(1));
assert_eq!(config.total_attempt_budget(), Some(1));
}
#[test]
fn max_retries_three_allows_four_total_attempts() {
let config = ReconnectConfig::default().max_retries(3);
assert!(!config.attempt_exhausted(3));
assert!(config.attempt_exhausted(4));
assert_eq!(config.total_attempt_budget(), Some(4));
}
#[test]
fn max_retries_none_never_fails() {
let config = ReconnectConfig::default(); let attempt = 9999usize;
assert!(!config.attempt_exhausted(attempt));
assert_eq!(config.total_attempt_budget(), None);
}
use std::future::poll_fn;
use std::sync::Mutex;
use tracing_subscriber::layer::{Context, Layer};
use tracing_subscriber::prelude::*;
#[derive(Clone, Default)]
struct EventCapture {
events: Arc<Mutex<Vec<String>>>,
}
struct FieldCollector(String);
impl tracing::field::Visit for FieldCollector {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.0.push_str(&format!(" {}={:?}", field.name(), value));
}
}
impl<S: tracing::Subscriber> Layer<S> for EventCapture {
fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
let mut collector = FieldCollector(String::new());
event.record(&mut collector);
self.events.lock().unwrap().push(collector.0);
}
}
#[tokio::test]
async fn reconnect_success_log_includes_duration() {
use tower_service::Service;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { while listener.accept().await.is_ok() {} });
let factory = move || async move {
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|e| RedisError::connection(addr.to_string(), e))?;
Ok::<_, RedisError>(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
};
let config = ReconnectConfig {
base_delay: Duration::from_millis(1),
max_delay: Duration::from_millis(1),
jitter: false,
..Default::default()
};
let lifecycle = ConnectionEventBus::new(8);
let mut lifecycle_stream = lifecycle.subscribe();
let mut conn = ResilientConnection::new_with_events(factory, config, lifecycle)
.await
.unwrap();
assert_eq!(
lifecycle_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
let capture = EventCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let _guard = tracing::subscriber::set_default(subscriber);
conn.needs_reconnect.store(true, Ordering::Release);
poll_fn(|cx| {
<ResilientConnection as Service<redis_tower_commands::Ping>>::poll_ready(&mut conn, cx)
})
.await
.expect("reconnect should succeed against the loopback listener");
assert_eq!(
lifecycle_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::from_millis(1),
}
);
assert!(matches!(
lifecycle_stream.recv().await.unwrap(),
ConnectionEvent::Reconnected { attempts: 1, .. }
));
let events = capture.events.lock().unwrap();
assert!(
events
.iter()
.any(|e| e.contains("reconnected successfully") && e.contains("elapsed_ms")),
"expected a reconnect success log carrying elapsed_ms, got: {events:?}"
);
}
}