use std::future::Future;
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use redis_tower_core::{Frame, ReceivedPushFrame, RedisConnection, RedisError};
use tokio::sync::{mpsc, oneshot, watch};
use tokio_util::sync::PollSender;
use tower_service::Service;
use tracing::warn;
use crate::maintenance::{
MaintenanceControl, MaintenanceListenerHandle, MaintenanceState, ParsedMaintenanceNotification,
next_listener_id, parse_maintenance_push, register_connection,
};
use crate::metrics_layer::MetricsRecorder;
use crate::reconnect::{
ConnectionDisconnectReason, ConnectionEvent, ConnectionEventBus, ConnectionFactory,
ReconnectConfig, connect_with_timeout,
};
#[derive(Clone)]
pub struct AutoPipelineConfig {
pub max_batch_size: usize,
pub batch_window: Duration,
pub queue_capacity: usize,
pub shed_load_on_full: bool,
pub response_timeout: Option<Duration>,
pub metrics_recorder: Option<Arc<dyn MetricsRecorder>>,
pub reconnect_on_readonly: bool,
}
impl std::fmt::Debug for AutoPipelineConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AutoPipelineConfig")
.field("max_batch_size", &self.max_batch_size)
.field("batch_window", &self.batch_window)
.field("queue_capacity", &self.queue_capacity)
.field("shed_load_on_full", &self.shed_load_on_full)
.field("response_timeout", &self.response_timeout)
.field(
"metrics_recorder",
&self.metrics_recorder.as_ref().map(|_| "<recorder>"),
)
.field("reconnect_on_readonly", &self.reconnect_on_readonly)
.finish()
}
}
impl Default for AutoPipelineConfig {
fn default() -> Self {
Self {
max_batch_size: 100,
batch_window: Duration::ZERO,
queue_capacity: 1024,
shed_load_on_full: false,
response_timeout: None,
metrics_recorder: None,
reconnect_on_readonly: false,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct AutoPipelineReconnectConfig {
pub reconnect: ReconnectConfig,
}
impl AutoPipelineReconnectConfig {
pub fn new(reconnect: ReconnectConfig) -> Self {
Self { reconnect }
}
}
enum ConnSource {
Fixed,
Factory {
factory: Arc<dyn ConnectionFactory>,
reconnect: ReconnectConfig,
},
}
struct MaintenanceWorker {
state: MaintenanceState,
control_rx: mpsc::UnboundedReceiver<MaintenanceControl>,
push_rx: tokio::sync::broadcast::Receiver<ReceivedPushFrame>,
}
enum PipelineFailure {
Connection(RedisError),
CommandTimeout,
ReadOnly,
}
impl PipelineFailure {
fn event_reason(&self) -> ConnectionDisconnectReason {
match self {
Self::Connection(error) => ConnectionDisconnectReason::ConnectionError {
error: Arc::from(error.to_string()),
},
Self::CommandTimeout => ConnectionDisconnectReason::CommandTimeout,
Self::ReadOnly => ConnectionDisconnectReason::ReadOnly,
}
}
}
enum WorkerRequest {
Single {
frame: Frame,
response_tx: oneshot::Sender<Result<Frame, RedisError>>,
},
Multi {
frames: Vec<Frame>,
response_tx: oneshot::Sender<Result<Vec<Frame>, RedisError>>,
},
}
impl WorkerRequest {
fn frame_count(&self) -> usize {
match self {
WorkerRequest::Single { .. } => 1,
WorkerRequest::Multi { frames, .. } => frames.len(),
}
}
fn fail(self, err: RedisError) {
match self {
WorkerRequest::Single { response_tx, .. } => {
let _ = response_tx.send(Err(err));
}
WorkerRequest::Multi { response_tx, .. } => {
let _ = response_tx.send(Err(err));
}
}
}
fn response_is_closed(&self) -> bool {
match self {
WorkerRequest::Single { response_tx, .. } => response_tx.is_closed(),
WorkerRequest::Multi { response_tx, .. } => response_tx.is_closed(),
}
}
}
pub struct AutoPipelineService {
tx: mpsc::Sender<WorkerRequest>,
poll_tx: PollSender<WorkerRequest>,
shed_load: bool,
lease: WorkerLease,
connection_health: watch::Receiver<bool>,
worker: Arc<WorkerHandle>,
}
struct WorkerControl {
state: Mutex<WorkerControlState>,
shutdown_tx: watch::Sender<bool>,
event_bus: Option<ConnectionEventBus>,
}
struct WorkerControlState {
handles: usize,
worker_running: bool,
shutdown_published: bool,
}
impl WorkerControl {
fn new(shutdown_tx: watch::Sender<bool>, event_bus: Option<ConnectionEventBus>) -> Self {
Self {
state: Mutex::new(WorkerControlState {
handles: 1,
worker_running: true,
shutdown_published: false,
}),
shutdown_tx,
event_bus,
}
}
fn add_handle(&self) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.handles = state
.handles
.checked_add(1)
.expect("AutoPipelineService handle count overflowed");
}
fn release_handle(&self) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.handles = state
.handles
.checked_sub(1)
.expect("AutoPipelineService handle count underflowed");
if state.handles == 0 {
self.shutdown_tx.send_replace(true);
if !state.worker_running {
self.publish_shutdown_locked(&mut state);
}
}
}
fn worker_finished(&self) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.worker_running = false;
if state.handles == 0 {
self.publish_shutdown_locked(&mut state);
}
}
fn publish_shutdown_locked(&self, state: &mut WorkerControlState) {
if state.shutdown_published {
return;
}
state.shutdown_published = true;
if let Some(events) = &self.event_bus {
events.publish(ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
});
}
}
fn with_active_handle<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.handles == 0 { None } else { Some(f()) }
}
}
struct WorkerRunGuard {
control: Arc<WorkerControl>,
}
impl Drop for WorkerRunGuard {
fn drop(&mut self) {
self.control.worker_finished();
}
}
struct WorkerLifecycle {
control: Arc<WorkerControl>,
shutdown: watch::Receiver<bool>,
connection_health: ConnectionHealthPublisher,
_run_guard: WorkerRunGuard,
}
struct ConnectionHealthPublisher {
tx: watch::Sender<bool>,
}
impl ConnectionHealthPublisher {
fn new(tx: watch::Sender<bool>) -> Self {
Self { tx }
}
fn set(&self, healthy: bool) {
self.tx.send_if_modified(|current| {
let changed = *current != healthy;
*current = healthy;
changed
});
}
}
impl Drop for ConnectionHealthPublisher {
fn drop(&mut self) {
self.set(false);
}
}
struct WorkerLease {
control: Arc<WorkerControl>,
}
impl Clone for WorkerLease {
fn clone(&self) -> Self {
self.control.add_handle();
Self {
control: Arc::clone(&self.control),
}
}
}
impl Drop for WorkerLease {
fn drop(&mut self) {
self.control.release_handle();
}
}
struct WorkerHandle {
handle: Option<tokio::task::JoinHandle<()>>,
}
impl WorkerHandle {
fn new(handle: tokio::task::JoinHandle<()>) -> Self {
Self {
handle: Some(handle),
}
}
fn is_finished(&self) -> bool {
self.handle
.as_ref()
.map(|h| h.is_finished())
.unwrap_or(true)
}
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
if !self.is_finished() {
warn!(
"AutoPipelineService dropped without calling shutdown(); \
background worker may still have queued requests in flight"
);
}
}
}
impl AutoPipelineService {
pub fn new(conn: RedisConnection, config: AutoPipelineConfig) -> Self {
Self::from_parts(Some(conn), config, ConnSource::Fixed, None, None)
}
pub fn new_with_events(
conn: RedisConnection,
config: AutoPipelineConfig,
events: ConnectionEventBus,
) -> Self {
Self::from_parts(Some(conn), config, ConnSource::Fixed, Some(events), None)
}
pub async fn with_factory(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
) -> Result<Self, RedisError> {
Self::with_factory_inner(factory, config, reconnect, None).await
}
pub async fn with_factory_and_events(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
events: ConnectionEventBus,
) -> Result<Self, RedisError> {
Self::with_factory_inner(factory, config, reconnect, Some(events)).await
}
pub async fn with_factory_and_maintenance(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
Self::with_factory_and_maintenance_inner(factory, config, reconnect, None).await
}
pub async fn with_factory_and_maintenance_and_events(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
events: ConnectionEventBus,
) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
Self::with_factory_and_maintenance_inner(factory, config, reconnect, Some(events)).await
}
async fn with_factory_and_maintenance_inner(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
event_bus: Option<ConnectionEventBus>,
) -> Result<(Self, MaintenanceListenerHandle), RedisError> {
let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
let (conn, push_rx) =
match connect_and_prepare(factory.as_ref(), reconnect.reconnect.connect_timeout, true)
.await
{
Ok((conn, Some(push_rx))) => (conn, push_rx),
Ok((_conn, None)) => {
unreachable!("maintenance setup always returns a push receiver")
}
Err(error) => {
if let Some(events) = &event_bus {
events.publish_with(|| ConnectionEvent::ConnectFailed {
error: Arc::from(error.to_string()),
});
}
return Err(error);
}
};
let listener_id = next_listener_id();
let (control_tx, control_rx) = mpsc::unbounded_channel();
let source = ConnSource::Factory {
factory,
reconnect: reconnect.reconnect,
};
let maintenance = MaintenanceWorker {
state: MaintenanceState::new(listener_id),
control_rx,
push_rx,
};
let service = Self::from_parts(Some(conn), config, source, event_bus, Some(maintenance));
let handle = MaintenanceListenerHandle::new(control_tx, listener_id);
Ok((service, handle))
}
async fn with_factory_inner(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
event_bus: Option<ConnectionEventBus>,
) -> Result<Self, RedisError> {
let factory: Arc<dyn ConnectionFactory> = Arc::new(factory);
let conn = match connect_with_timeout(factory.as_ref(), reconnect.reconnect.connect_timeout)
.await
{
Ok(conn) => conn,
Err(error) => {
if let Some(events) = &event_bus {
events.publish_with(|| ConnectionEvent::ConnectFailed {
error: Arc::from(error.to_string()),
});
}
return Err(error);
}
};
let source = ConnSource::Factory {
factory,
reconnect: reconnect.reconnect,
};
Ok(Self::from_parts(
Some(conn),
config,
source,
event_bus,
None,
))
}
pub fn with_lazy_factory(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
) -> Self {
Self::with_lazy_factory_inner(factory, config, reconnect, None)
}
pub fn with_lazy_factory_and_events(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
events: ConnectionEventBus,
) -> Self {
Self::with_lazy_factory_inner(factory, config, reconnect, Some(events))
}
fn with_lazy_factory_inner(
factory: impl ConnectionFactory,
config: AutoPipelineConfig,
reconnect: AutoPipelineReconnectConfig,
event_bus: Option<ConnectionEventBus>,
) -> Self {
let source = ConnSource::Factory {
factory: Arc::new(factory),
reconnect: reconnect.reconnect,
};
Self::from_parts(None, config, source, event_bus, None)
}
fn from_parts(
conn: Option<RedisConnection>,
config: AutoPipelineConfig,
source: ConnSource,
event_bus: Option<ConnectionEventBus>,
maintenance: Option<MaintenanceWorker>,
) -> Self {
let (tx, rx) = mpsc::channel(config.queue_capacity);
let poll_tx = PollSender::new(tx.clone());
let shed_load = config.shed_load_on_full;
let (shutdown_tx, shutdown_rx) = watch::channel(false);
let initially_connected = conn.is_some();
let (connection_health_tx, connection_health) = watch::channel(initially_connected);
let control = Arc::new(WorkerControl::new(shutdown_tx, event_bus.clone()));
let lease = WorkerLease {
control: Arc::clone(&control),
};
if initially_connected && let Some(events) = &event_bus {
events.publish(ConnectionEvent::Connected);
}
let lifecycle = WorkerLifecycle {
control: Arc::clone(&control),
shutdown: shutdown_rx,
connection_health: ConnectionHealthPublisher::new(connection_health_tx),
_run_guard: WorkerRunGuard {
control: Arc::clone(&control),
},
};
let runtime = tokio::runtime::Handle::try_current()
.expect("AutoPipelineService must be constructed inside an entered Tokio runtime");
let handle = runtime.spawn(pipeline_worker(
rx,
conn,
config,
source,
event_bus,
lifecycle,
maintenance,
));
Self {
tx,
poll_tx,
shed_load,
lease,
connection_health,
worker: Arc::new(WorkerHandle::new(handle)),
}
}
pub async fn call_pipeline(&mut self, frames: Vec<Frame>) -> Result<Vec<Frame>, RedisError> {
if frames.is_empty() {
return Ok(Vec::new());
}
let (resp_tx, resp_rx) = oneshot::channel();
let request = WorkerRequest::Multi {
frames,
response_tx: resp_tx,
};
if self.shed_load {
self.tx.try_send(request).map_err(|e| match e {
tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
tokio::sync::mpsc::error::TrySendError::Closed(_) => RedisError::ConnectionClosed,
})?;
} else {
self.tx
.reserve()
.await
.map_err(|_| RedisError::ConnectionClosed)?
.send(request);
}
resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
}
pub(crate) fn call_reserved_pipeline(
&mut self,
frames: Vec<Frame>,
) -> Pin<Box<dyn Future<Output = Result<Vec<Frame>, RedisError>> + Send>> {
if frames.is_empty() {
self.release_reservation();
return Box::pin(async { Ok(Vec::new()) });
}
let (resp_tx, resp_rx) = oneshot::channel();
let request = WorkerRequest::Multi {
frames,
response_tx: resp_tx,
};
let send_result = if self.shed_load {
self.tx.try_send(request).map_err(|e| match e {
tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
tokio::sync::mpsc::error::TrySendError::Closed(_) => RedisError::ConnectionClosed,
})
} else {
self.poll_tx
.send_item(request)
.map_err(|_| RedisError::ConnectionClosed)
};
Box::pin(async move {
send_result?;
resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
})
}
pub(crate) fn release_reservation(&mut self) {
let _ = self.poll_tx.abort_send();
}
pub async fn shutdown(self) {
drop(self.tx);
drop(self.poll_tx);
drop(self.lease);
if let Ok(mut worker_handle) = Arc::try_unwrap(self.worker)
&& let Some(handle) = worker_handle.handle.take()
{
let _ = handle.await;
}
}
pub fn is_alive(&self) -> bool {
!self.worker.is_finished()
}
#[doc(hidden)]
pub fn is_connection_healthy(&self) -> bool {
*self.connection_health.borrow()
}
#[doc(hidden)]
pub fn subscribe_connection_health(&self) -> watch::Receiver<bool> {
self.connection_health.clone()
}
}
async fn pipeline_worker(
mut rx: mpsc::Receiver<WorkerRequest>,
mut conn: Option<RedisConnection>,
config: AutoPipelineConfig,
source: ConnSource,
event_bus: Option<ConnectionEventBus>,
lifecycle: WorkerLifecycle,
maintenance: Option<MaintenanceWorker>,
) {
let WorkerLifecycle {
control,
mut shutdown,
connection_health,
_run_guard,
} = lifecycle;
let mut outage_reported = false;
let mut shutting_down = false;
let (mut maintenance_state, mut maintenance_control_rx, mut maintenance_push_rx) =
match maintenance {
Some(maintenance) => (
Some(maintenance.state),
Some(maintenance.control_rx),
Some(maintenance.push_rx),
),
None => (None, None, None),
};
'worker: loop {
drain_maintenance_control(
&mut maintenance_state,
&mut maintenance_control_rx,
&mut maintenance_push_rx,
);
if maintenance_state
.as_ref()
.and_then(|state| state.pending.as_ref())
.is_some()
{
match wait_for_pending_handoff(
&mut conn,
&mut maintenance_state,
&mut maintenance_control_rx,
&mut maintenance_push_rx,
event_bus.as_ref(),
&mut shutdown,
)
.await
{
PendingHandoffOutcome::Cancelled => continue,
PendingHandoffOutcome::Shutdown => {
shutting_down = true;
rx.close();
disable_maintenance(
&mut maintenance_state,
&mut maintenance_control_rx,
&mut maintenance_push_rx,
);
continue;
}
PendingHandoffOutcome::ConnectionFailed(error) => {
clear_pending_handoff(&mut maintenance_state);
connection_health.set(false);
let failure = PipelineFailure::Connection(error);
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
failure.event_reason(),
&control,
);
let ConnSource::Factory { factory, reconnect } = &source else {
fail_queued_requests(&mut rx);
return;
};
let maintenance_enabled = maintenance_state
.as_ref()
.is_some_and(|state| state.enabled);
if !reconnect_worker_connection(
conn.as_mut()
.expect("maintenance listener requires a connection"),
&mut rx,
factory.as_ref(),
reconnect,
event_bus.as_ref(),
&mut shutdown,
&control,
&connection_health,
&mut outage_reported,
&mut maintenance_push_rx,
maintenance_enabled,
)
.await
{
return;
}
continue;
}
PendingHandoffOutcome::Ready { sequence } => {
if let Some(state) = &mut maintenance_state {
state.pending = None;
}
connection_health.set(false);
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
ConnectionDisconnectReason::MaintenanceHandoff { sequence },
&control,
);
let ConnSource::Factory { factory, reconnect } = &source else {
fail_queued_requests(&mut rx);
return;
};
let maintenance_enabled = maintenance_state
.as_ref()
.is_some_and(|state| state.enabled);
if !reconnect_worker_connection(
conn.as_mut()
.expect("maintenance listener requires a connection"),
&mut rx,
factory.as_ref(),
reconnect,
event_bus.as_ref(),
&mut shutdown,
&control,
&connection_health,
&mut outage_reported,
&mut maintenance_push_rx,
maintenance_enabled,
)
.await
{
return;
}
continue;
}
}
}
let first = if shutting_down {
rx.recv().await
} else {
loop {
tokio::select! {
biased;
() = wait_for_shutdown(&mut shutdown) => {
shutting_down = true;
rx.close();
break rx.recv().await;
}
control_message = recv_maintenance_control(&mut maintenance_control_rx) => {
apply_maintenance_control(
control_message,
&mut maintenance_state,
&mut maintenance_control_rx,
&mut maintenance_push_rx,
);
}
push = recv_maintenance_push(&mut maintenance_push_rx) => {
if handle_maintenance_push_result(
push,
&mut maintenance_state,
&mut maintenance_push_rx,
event_bus.as_ref(),
).is_err() {
connection_health.set(false);
let failure = PipelineFailure::Connection(
maintenance_feed_error(),
);
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
failure.event_reason(),
&control,
);
let ConnSource::Factory { factory, reconnect } = &source else {
fail_queued_requests(&mut rx);
return;
};
let maintenance_enabled = maintenance_state
.as_ref()
.is_some_and(|state| state.enabled);
if !reconnect_worker_connection(
conn.as_mut().expect("maintenance feed requires a connection"),
&mut rx,
factory.as_ref(),
reconnect,
event_bus.as_ref(),
&mut shutdown,
&control,
&connection_health,
&mut outage_reported,
&mut maintenance_push_rx,
maintenance_enabled,
).await {
return;
}
}
if maintenance_state
.as_ref()
.and_then(|state| state.pending.as_ref())
.is_some()
{
continue 'worker;
}
}
request = rx.recv() => break request,
idle_result = read_idle_push_if_connected(&mut conn) => match idle_result {
Ok(()) => {
}
Err(error) => {
clear_pending_handoff(&mut maintenance_state);
connection_health.set(false);
let failure = PipelineFailure::Connection(error);
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
failure.event_reason(),
&control,
);
match &source {
ConnSource::Fixed => {
rx.close();
while let Ok(request) = rx.try_recv() {
request.fail(RedisError::ConnectionClosed);
}
return;
}
ConnSource::Factory { factory, reconnect } => {
let maintenance_enabled = maintenance_state
.as_ref()
.is_some_and(|state| state.enabled);
if !reconnect_worker_connection(
conn.as_mut().expect("idle failure requires a connection"),
&mut rx,
factory.as_ref(),
reconnect,
event_bus.as_ref(),
&mut shutdown,
&control,
&connection_health,
&mut outage_reported,
&mut maintenance_push_rx,
maintenance_enabled,
)
.await
{
return;
}
}
}
}
},
}
}
};
let first = match first {
Some(req) => req,
None => break, };
let mut frame_count = first.frame_count();
let mut batch: Vec<WorkerRequest> = vec![first];
while frame_count < config.max_batch_size {
match rx.try_recv() {
Ok(req) => {
frame_count += req.frame_count();
batch.push(req);
}
Err(_) => break,
}
}
let mut channel_closed = false;
if frame_count < config.max_batch_size && !config.batch_window.is_zero() {
let deadline = tokio::time::Instant::now() + config.batch_window;
loop {
if frame_count >= config.max_batch_size {
break;
}
if shutting_down {
match rx.try_recv() {
Ok(req) => {
frame_count += req.frame_count();
batch.push(req);
continue;
}
Err(_) => break,
}
}
tokio::select! {
biased;
() = wait_for_shutdown(&mut shutdown) => {
shutting_down = true;
rx.close();
}
request = rx.recv() => match request {
Some(req) => {
frame_count += req.frame_count();
batch.push(req);
}
None => {
channel_closed = true;
break;
}
},
() = tokio::time::sleep_until(deadline) => break,
}
}
}
if conn.is_none() {
let ConnSource::Factory { factory, reconnect } = &source else {
unreachable!("a fixed worker always starts with a connection");
};
let connect = connect_with_timeout(factory.as_ref(), reconnect.connect_timeout);
tokio::pin!(connect);
let connect_result = tokio::select! {
biased;
() = wait_for_shutdown(&mut shutdown) => {
shutting_down = true;
rx.close();
fail_requests(batch);
continue;
}
result = &mut connect => result,
};
match connect_result {
Ok(new_conn) => {
let connected = control.with_active_handle(|| {
connection_health.set(true);
if let Some(events) = &event_bus {
events.publish(ConnectionEvent::Connected);
}
});
if connected.is_none() {
fail_requests(batch);
return;
}
conn = Some(new_conn);
}
Err(error) => {
control.with_active_handle(|| {
if let Some(events) = &event_bus {
events.publish_with(|| ConnectionEvent::ConnectFailed {
error: Arc::from(error.to_string()),
});
}
});
fail_requests(batch);
continue;
}
}
}
let flush_result = flush_batch(
conn.as_mut().expect("connection established before flush"),
batch,
config.response_timeout,
config.metrics_recorder.as_ref(),
config.reconnect_on_readonly,
&connection_health,
)
.await;
if channel_closed {
if let Err(failure) = flush_result {
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
failure.event_reason(),
&control,
);
}
return;
}
if let Err(failure) = flush_result {
drain_maintenance_control(
&mut maintenance_state,
&mut maintenance_control_rx,
&mut maintenance_push_rx,
);
drain_available_maintenance_pushes(
&mut maintenance_state,
&mut maintenance_push_rx,
event_bus.as_ref(),
);
clear_pending_handoff(&mut maintenance_state);
publish_worker_outage(
event_bus.as_ref(),
&mut outage_reported,
failure.event_reason(),
&control,
);
match &source {
ConnSource::Fixed => {
}
ConnSource::Factory { factory, reconnect } => {
let maintenance_enabled = maintenance_state
.as_ref()
.is_some_and(|state| state.enabled);
if !reconnect_worker_connection(
conn.as_mut().expect("failed flush requires a connection"),
&mut rx,
factory.as_ref(),
reconnect,
event_bus.as_ref(),
&mut shutdown,
&control,
&connection_health,
&mut outage_reported,
&mut maintenance_push_rx,
maintenance_enabled,
)
.await
{
return;
}
}
}
}
}
}
async fn read_idle_push_if_connected(conn: &mut Option<RedisConnection>) -> Result<(), RedisError> {
match conn {
Some(conn) => conn.read_idle_push().await,
None => futures::future::pending().await,
}
}
enum PendingHandoffOutcome {
Cancelled,
Ready { sequence: u64 },
ConnectionFailed(RedisError),
Shutdown,
}
async fn recv_maintenance_control(
receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
) -> Option<MaintenanceControl> {
match receiver {
Some(receiver) => receiver.recv().await,
None => futures::future::pending().await,
}
}
async fn recv_maintenance_push(
receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
) -> Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError> {
match receiver {
Some(receiver) => receiver.recv().await,
None => futures::future::pending().await,
}
}
fn drain_maintenance_control(
state: &mut Option<MaintenanceState>,
receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
) {
loop {
let Some(control) = receiver.as_mut() else {
return;
};
match control.try_recv() {
Ok(message) => apply_maintenance_control(Some(message), state, receiver, pushes),
Err(mpsc::error::TryRecvError::Empty) => return,
Err(mpsc::error::TryRecvError::Disconnected) => {
*receiver = None;
if let Some(state) = state {
state.enabled = false;
state.pending = None;
}
*pushes = None;
return;
}
}
}
}
fn clear_pending_handoff(state: &mut Option<MaintenanceState>) {
if let Some(state) = state {
state.pending = None;
}
}
fn disable_maintenance(
state: &mut Option<MaintenanceState>,
receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
) {
if let Some(state) = state {
state.enabled = false;
state.pending = None;
}
*receiver = None;
*pushes = None;
}
fn apply_maintenance_control(
message: Option<MaintenanceControl>,
state: &mut Option<MaintenanceState>,
receiver: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
pushes: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
) {
let Some(message) = message else {
*receiver = None;
return;
};
match message {
MaintenanceControl::Disable { listener_id, ack } => {
if state
.as_ref()
.is_some_and(|state| state.listener_id == listener_id)
{
disable_maintenance(state, receiver, pushes);
}
if let Some(ack) = ack {
let _ = ack.send(());
}
}
}
}
fn publish_maintenance_notification(
event_bus: Option<&ConnectionEventBus>,
notification: ParsedMaintenanceNotification,
) {
if let Some(events) = event_bus {
events.publish(ConnectionEvent::MaintenanceNotification {
kind: notification.kind(),
sequence: notification.sequence(),
ttl: notification.ttl(),
});
}
}
fn handle_maintenance_push_result(
push: Result<ReceivedPushFrame, tokio::sync::broadcast::error::RecvError>,
state: &mut Option<MaintenanceState>,
receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
event_bus: Option<&ConnectionEventBus>,
) -> Result<(), ()> {
match push {
Ok(push) => {
if let Some(state) = state
&& let Some(received) = parse_maintenance_push(push)
&& let Some(notification) = state.accept(received)
{
publish_maintenance_notification(event_bus, notification);
}
Ok(())
}
Err(_) => {
*receiver = None;
Err(())
}
}
}
fn drain_available_maintenance_pushes(
state: &mut Option<MaintenanceState>,
receiver: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
event_bus: Option<&ConnectionEventBus>,
) {
loop {
let Some(pushes) = receiver.as_mut() else {
return;
};
match pushes.try_recv() {
Ok(push) => {
let _ = handle_maintenance_push_result(Ok(push), state, receiver, event_bus);
}
Err(tokio::sync::broadcast::error::TryRecvError::Empty) => return,
Err(tokio::sync::broadcast::error::TryRecvError::Lagged(_)) => {
}
Err(tokio::sync::broadcast::error::TryRecvError::Closed) => {
*receiver = None;
return;
}
}
}
}
fn maintenance_feed_error() -> RedisError {
RedisError::UnexpectedResponse {
expected: "a complete maintenance-notification push feed",
actual: "maintenance push feed closed or overflowed".to_owned(),
}
}
async fn connect_and_prepare(
factory: &dyn ConnectionFactory,
timeout: Option<Duration>,
maintenance_enabled: bool,
) -> Result<
(
RedisConnection,
Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
),
RedisError,
> {
let prepare = async {
let mut connection = factory.connect().await?;
let pushes = if maintenance_enabled {
Some(register_connection(&mut connection).await?)
} else {
None
};
Ok((connection, pushes))
};
match timeout {
Some(timeout) => tokio::time::timeout(timeout, prepare)
.await
.map_err(|_| RedisError::ConnectTimeout)?,
None => prepare.await,
}
}
async fn wait_for_pending_handoff(
conn: &mut Option<RedisConnection>,
state: &mut Option<MaintenanceState>,
control_rx: &mut Option<mpsc::UnboundedReceiver<MaintenanceControl>>,
push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
event_bus: Option<&ConnectionEventBus>,
shutdown: &mut watch::Receiver<bool>,
) -> PendingHandoffOutcome {
loop {
let Some(pending) = state.as_ref().and_then(|state| state.pending.as_ref()) else {
return PendingHandoffOutcome::Cancelled;
};
let deadline = pending.deadline;
let sequence = pending.sequence;
if deadline <= tokio::time::Instant::now() {
return PendingHandoffOutcome::Ready { sequence };
}
tokio::select! {
biased;
() = wait_for_shutdown(shutdown) => return PendingHandoffOutcome::Shutdown,
control = recv_maintenance_control(control_rx) => {
apply_maintenance_control(control, state, control_rx, push_rx);
if state.as_ref().and_then(|state| state.pending.as_ref()).is_none() {
return PendingHandoffOutcome::Cancelled;
}
}
push = recv_maintenance_push(push_rx) => {
if handle_maintenance_push_result(push, state, push_rx, event_bus).is_err() {
return PendingHandoffOutcome::ConnectionFailed(maintenance_feed_error());
}
}
result = read_idle_push_if_connected(conn) => {
if let Err(error) = result {
return PendingHandoffOutcome::ConnectionFailed(error);
}
}
() = tokio::time::sleep_until(deadline) => {
}
}
}
}
fn fail_requests(requests: Vec<WorkerRequest>) {
for request in requests {
request.fail(RedisError::ConnectionClosed);
}
}
#[allow(clippy::too_many_arguments)]
async fn reconnect_worker_connection(
conn: &mut RedisConnection,
rx: &mut mpsc::Receiver<WorkerRequest>,
factory: &dyn ConnectionFactory,
reconnect: &ReconnectConfig,
event_bus: Option<&ConnectionEventBus>,
shutdown: &mut watch::Receiver<bool>,
control: &WorkerControl,
connection_health: &ConnectionHealthPublisher,
outage_reported: &mut bool,
maintenance_push_rx: &mut Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
maintenance_enabled: bool,
) -> bool {
let started = Instant::now();
match reconnect_with_backoff(
factory,
reconnect,
event_bus,
shutdown,
control,
maintenance_enabled,
)
.await
{
ReconnectOutcome::Connected(new_conn, new_push_rx, attempts) => {
let reconnected = control.with_active_handle(|| {
*conn = *new_conn;
*maintenance_push_rx = new_push_rx;
connection_health.set(true);
*outage_reported = false;
if let Some(events) = event_bus {
events.publish(ConnectionEvent::Reconnected {
attempts,
elapsed: started.elapsed(),
});
}
});
if reconnected.is_some() {
true
} else {
fail_queued_requests(rx);
false
}
}
ReconnectOutcome::Exhausted | ReconnectOutcome::Shutdown => {
fail_queued_requests(rx);
false
}
}
}
fn fail_queued_requests(rx: &mut mpsc::Receiver<WorkerRequest>) {
rx.close();
while let Ok(request) = rx.try_recv() {
request.fail(RedisError::ConnectionClosed);
}
}
fn publish_worker_outage(
event_bus: Option<&ConnectionEventBus>,
outage_reported: &mut bool,
reason: ConnectionDisconnectReason,
control: &WorkerControl,
) {
if *outage_reported {
return;
}
let active = control.with_active_handle(|| {
if let Some(events) = event_bus {
events.publish(ConnectionEvent::Disconnected { reason });
}
});
if active.is_some() {
*outage_reported = true;
}
}
fn is_readonly_frame(frame: &Frame) -> bool {
matches!(frame, Frame::Error(e) if e.len() >= 8 && e[..8].eq_ignore_ascii_case(b"READONLY"))
}
async fn flush_batch(
conn: &mut RedisConnection,
batch: Vec<WorkerRequest>,
response_timeout: Option<Duration>,
recorder: Option<&Arc<dyn MetricsRecorder>>,
reconnect_on_readonly: bool,
connection_health: &ConnectionHealthPublisher,
) -> Result<(), PipelineFailure> {
let mut batch = batch;
batch.retain(|request| !request.response_is_closed());
if batch.is_empty() {
return Ok(());
}
let total_frames: usize = batch.iter().map(|r| r.frame_count()).sum();
if let Some(recorder) = recorder {
recorder.pipeline_flushed(total_frames);
}
let mut frames: Vec<Frame> = Vec::with_capacity(total_frames);
enum Responder {
Single(oneshot::Sender<Result<Frame, RedisError>>),
Multi(usize, oneshot::Sender<Result<Vec<Frame>, RedisError>>),
}
let mut responders: Vec<Responder> = Vec::with_capacity(batch.len());
for req in batch {
match req {
WorkerRequest::Single { frame, response_tx } => {
frames.push(frame);
responders.push(Responder::Single(response_tx));
}
WorkerRequest::Multi {
frames: fs,
response_tx,
} => {
let count = fs.len();
frames.extend(fs);
responders.push(Responder::Multi(count, response_tx));
}
}
}
let exec_result = match response_timeout {
Some(timeout) => match tokio::time::timeout(timeout, conn.execute_pipeline(frames)).await {
Ok(result) => result,
Err(_elapsed) => {
connection_health.set(false);
for responder in responders {
match responder {
Responder::Single(tx) => {
let _ = tx.send(Err(RedisError::CommandTimeout));
}
Responder::Multi(_, tx) => {
let _ = tx.send(Err(RedisError::CommandTimeout));
}
}
}
return Err(PipelineFailure::CommandTimeout);
}
},
None => conn.execute_pipeline(frames).await,
};
match exec_result {
Ok(responses) => {
let saw_readonly = reconnect_on_readonly && responses.iter().any(is_readonly_frame);
if saw_readonly {
connection_health.set(false);
}
let mut iter = responses.into_iter();
for responder in responders {
match responder {
Responder::Single(tx) => {
let _ = tx.send(iter.next().ok_or(RedisError::ConnectionClosed));
}
Responder::Multi(count, tx) => {
let collected: Vec<Frame> = iter.by_ref().take(count).collect();
if collected.len() == count {
let _ = tx.send(Ok(collected));
} else {
let _ = tx.send(Err(RedisError::ConnectionClosed));
}
}
}
}
if saw_readonly {
Err(PipelineFailure::ReadOnly)
} else {
Ok(())
}
}
Err(error) => {
connection_health.set(false);
for responder in responders {
match responder {
Responder::Single(tx) => {
let _ = tx.send(Err(RedisError::ConnectionClosed));
}
Responder::Multi(_, tx) => {
let _ = tx.send(Err(RedisError::ConnectionClosed));
}
}
}
Err(PipelineFailure::Connection(error))
}
}
}
enum ReconnectOutcome {
Connected(
Box<RedisConnection>,
Option<tokio::sync::broadcast::Receiver<ReceivedPushFrame>>,
usize,
),
Exhausted,
Shutdown,
}
async fn wait_for_shutdown(shutdown: &mut watch::Receiver<bool>) {
if *shutdown.borrow() {
return;
}
let _ = shutdown.changed().await;
}
async fn reconnect_with_backoff(
factory: &dyn ConnectionFactory,
config: &ReconnectConfig,
event_bus: Option<&ConnectionEventBus>,
shutdown: &mut watch::Receiver<bool>,
control: &WorkerControl,
maintenance_enabled: bool,
) -> ReconnectOutcome {
let mut attempt: usize = 0;
loop {
if *shutdown.borrow() {
return ReconnectOutcome::Shutdown;
}
if config.attempt_exhausted(attempt) {
warn!(
attempts = attempt,
"auto_pipeline: reconnect attempts exhausted"
);
let active = control.with_active_handle(|| {
if let Some(events) = event_bus {
events.publish(ConnectionEvent::ReconnectExhausted { attempts: attempt });
}
});
if active.is_none() {
return ReconnectOutcome::Shutdown;
}
return ReconnectOutcome::Exhausted;
}
let delay = config.delay_for_attempt(attempt);
let active = control.with_active_handle(|| {
if let Some(events) = event_bus {
events.publish(ConnectionEvent::ReconnectAttempt {
attempt: attempt + 1,
delay,
});
}
});
if active.is_none() {
return ReconnectOutcome::Shutdown;
}
if !delay.is_zero() {
tokio::select! {
biased;
() = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
() = tokio::time::sleep(delay) => {}
}
}
attempt += 1;
let connect = connect_and_prepare(factory, config.connect_timeout, maintenance_enabled);
tokio::pin!(connect);
let result = tokio::select! {
biased;
() = wait_for_shutdown(shutdown) => return ReconnectOutcome::Shutdown,
result = &mut connect => result,
};
match result {
Ok((conn, push_rx)) => {
return ReconnectOutcome::Connected(Box::new(conn), push_rx, attempt);
}
Err(e) => {
warn!(attempt, error = %e, "auto_pipeline: reconnect attempt failed");
let active = control.with_active_handle(|| {
if let Some(events) = event_bus {
events.publish_with(|| ConnectionEvent::ReconnectFailed {
attempt,
error: Arc::from(e.to_string()),
});
}
});
if active.is_none() {
return ReconnectOutcome::Shutdown;
}
continue;
}
}
}
}
impl Service<Frame> for AutoPipelineService {
type Response = Frame;
type Error = RedisError;
type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
if self.shed_load {
return if self.tx.is_closed() {
Poll::Ready(Err(RedisError::ConnectionClosed))
} else {
Poll::Ready(Ok(()))
};
}
self.poll_tx
.poll_reserve(cx)
.map_err(|_| RedisError::ConnectionClosed)
}
fn call(&mut self, frame: Frame) -> Self::Future {
let (resp_tx, resp_rx) = oneshot::channel();
let request = WorkerRequest::Single {
frame,
response_tx: resp_tx,
};
if self.shed_load {
let send_result = self.tx.try_send(request).map_err(|e| match e {
tokio::sync::mpsc::error::TrySendError::Full(_) => RedisError::QueueFull,
tokio::sync::mpsc::error::TrySendError::Closed(_) => RedisError::ConnectionClosed,
});
return Box::pin(async move {
send_result?;
resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
});
}
let send_result = self
.poll_tx
.send_item(request)
.map_err(|_| RedisError::ConnectionClosed);
Box::pin(async move {
send_result?;
resp_rx.await.map_err(|_| RedisError::ConnectionClosed)?
})
}
}
impl Clone for AutoPipelineService {
fn clone(&self) -> Self {
Self {
tx: self.tx.clone(),
poll_tx: PollSender::new(self.tx.clone()),
shed_load: self.shed_load,
lease: self.lease.clone(),
connection_health: self.connection_health.clone(),
worker: Arc::clone(&self.worker),
}
}
}
impl AutoPipelineService {
pub fn queue_depth(&self) -> usize {
self.tx.max_capacity() - self.tx.capacity()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lazy_factory_requires_an_entered_tokio_runtime() {
let result = std::panic::catch_unwind(|| {
AutoPipelineService::with_lazy_factory(
|| async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) },
AutoPipelineConfig::default(),
AutoPipelineReconnectConfig::default(),
)
});
assert!(result.is_err());
}
#[tokio::test]
async fn lazy_factory_defers_connection_and_retries_after_initial_failure() {
use futures::{SinkExt, StreamExt};
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio_util::codec::Framed;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut framed = Framed::new(
redis_tower_core::RedisStream::Tcp(stream),
redis_tower_core::RespCodec::new(),
);
let request = framed
.next()
.await
.expect("client closed before request")
.expect("client sent an invalid request");
assert_eq!(
request,
Frame::SimpleString(bytes::Bytes::from_static(b"PING"))
);
framed
.send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
.await
.unwrap();
let _ = framed.next().await;
});
let calls = Arc::new(AtomicUsize::new(0));
let factory_calls = Arc::clone(&calls);
let factory = move || {
let factory_calls = Arc::clone(&factory_calls);
async move {
if factory_calls.fetch_add(1, Ordering::AcqRel) == 0 {
return Err(RedisError::ConnectionClosed);
}
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|error| RedisError::connection(addr.to_string(), error))?;
Ok(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
}
};
let events = ConnectionEventBus::new(8);
let mut event_stream = events.subscribe();
let mut service = AutoPipelineService::with_lazy_factory_and_events(
factory,
AutoPipelineConfig::default(),
AutoPipelineReconnectConfig::default(),
events,
);
assert_eq!(calls.load(Ordering::Acquire), 0);
assert!(!service.is_connection_healthy());
assert!(
tokio::time::timeout(Duration::from_millis(10), event_stream.recv())
.await
.is_err(),
"lazy construction must not publish a connection event"
);
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
assert!(matches!(
service
.call(Frame::SimpleString(bytes::Bytes::from_static(b"FIRST")))
.await,
Err(RedisError::ConnectionClosed)
));
assert!(!service.is_connection_healthy());
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ConnectFailed { .. }
));
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
assert_eq!(
service
.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
.await
.unwrap(),
Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
);
assert!(service.is_connection_healthy());
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
assert_eq!(calls.load(Ordering::Acquire), 2);
service.shutdown().await;
server.await.unwrap();
}
#[tokio::test]
async fn lazy_factory_shutdown_cancels_first_connection_attempt() {
use tokio::sync::Notify;
let entered = Arc::new(Notify::new());
let factory_entered = Arc::clone(&entered);
let factory = move || {
let factory_entered = Arc::clone(&factory_entered);
async move {
factory_entered.notify_one();
futures::future::pending::<Result<RedisConnection, RedisError>>().await
}
};
let mut service = AutoPipelineService::with_lazy_factory(
factory,
AutoPipelineConfig {
shed_load_on_full: true,
..AutoPipelineConfig::default()
},
AutoPipelineReconnectConfig::default(),
);
let response = service.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")));
entered.notified().await;
tokio::time::timeout(Duration::from_secs(1), service.shutdown())
.await
.expect("shutdown must cancel a deferred connection attempt");
assert!(matches!(response.await, Err(RedisError::ConnectionClosed)));
}
#[tokio::test]
async fn initial_factory_timeout_publishes_connect_failed() {
let factory =
|| async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let reconnect = AutoPipelineReconnectConfig::new(
ReconnectConfig::default().connect_timeout(Duration::from_millis(10)),
);
let result = AutoPipelineService::with_factory_and_events(
factory,
AutoPipelineConfig::default(),
reconnect,
events,
)
.await;
assert!(matches!(result, Err(RedisError::ConnectTimeout)));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ConnectFailed {
error: Arc::from(RedisError::ConnectTimeout.to_string()),
}
);
}
#[tokio::test]
async fn reconnect_factory_timeout_is_counted_and_exhausted() {
let factory =
|| async { futures::future::pending::<Result<RedisConnection, RedisError>>().await };
let config = ReconnectConfig {
max_retries: Some(0),
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: false,
connect_timeout: Some(Duration::from_millis(10)),
};
let events = ConnectionEventBus::new(4);
let mut stream = events.subscribe();
let (shutdown_tx, mut shutdown) = watch::channel(false);
let control = WorkerControl::new(shutdown_tx, None);
assert!(matches!(
reconnect_with_backoff(
&factory,
&config,
Some(&events),
&mut shutdown,
&control,
false,
)
.await,
ReconnectOutcome::Exhausted
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::ZERO,
}
);
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed {
attempt: 1,
error: Arc::from(RedisError::ConnectTimeout.to_string()),
}
);
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectExhausted { attempts: 1 }
);
}
#[tokio::test]
async fn reconnect_exhaustion_is_published_after_final_failure() {
let factory = || async { Err::<RedisConnection, _>(RedisError::ConnectionClosed) };
let config = ReconnectConfig {
max_retries: Some(2),
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: false,
connect_timeout: None,
};
let events = ConnectionEventBus::new(8);
let mut stream = events.subscribe();
let (shutdown_tx, mut shutdown) = watch::channel(false);
let control = WorkerControl::new(shutdown_tx, None);
assert!(matches!(
reconnect_with_backoff(
&factory,
&config,
Some(&events),
&mut shutdown,
&control,
false,
)
.await,
ReconnectOutcome::Exhausted
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::ZERO,
}
);
assert!(matches!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed { attempt: 1, .. }
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 2,
delay: Duration::ZERO,
}
);
assert!(matches!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed { attempt: 2, .. }
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 3,
delay: Duration::ZERO,
}
);
assert!(matches!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed { attempt: 3, .. }
));
assert_eq!(
stream.recv().await.unwrap(),
ConnectionEvent::ReconnectExhausted { attempts: 3 }
);
}
#[tokio::test]
async fn factory_worker_detects_idle_disconnect_and_reconnects_in_order() {
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::{Notify, oneshot};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (first_closed_tx, first_closed_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (first, _) = listener.accept().await.unwrap();
drop(first);
let _ = first_closed_tx.send(());
let (second, _) = listener.accept().await.unwrap();
let _second = second;
futures::future::pending::<()>().await;
});
let calls = Arc::new(AtomicUsize::new(0));
let allow_reconnect = Arc::new(Notify::new());
let factory_calls = Arc::clone(&calls);
let factory_allow_reconnect = Arc::clone(&allow_reconnect);
let factory = move || {
let factory_calls = Arc::clone(&factory_calls);
let factory_allow_reconnect = Arc::clone(&factory_allow_reconnect);
async move {
let call = factory_calls.fetch_add(1, Ordering::AcqRel);
if call == 1 {
return Err(RedisError::ConnectionClosed);
}
if call == 2 {
factory_allow_reconnect.notified().await;
}
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|error| RedisError::connection(addr.to_string(), error))?;
Ok(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
}
};
let events = ConnectionEventBus::new(16);
let mut event_stream = events.subscribe();
let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
max_retries: Some(3),
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: false,
connect_timeout: None,
});
let service = AutoPipelineService::with_factory_and_events(
factory,
AutoPipelineConfig::default(),
reconnect,
events,
)
.await
.unwrap();
let mut health = service.subscribe_connection_health();
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
first_closed_rx.await.unwrap();
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::ZERO,
}
);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed {
attempt: 1,
error: Arc::from("connection closed"),
}
);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 2,
delay: Duration::ZERO,
}
);
health.changed().await.unwrap();
assert!(!*health.borrow_and_update());
assert!(!service.is_connection_healthy());
allow_reconnect.notify_one();
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Reconnected { attempts: 2, .. }
));
health.changed().await.unwrap();
assert!(*health.borrow_and_update());
assert!(service.is_connection_healthy());
assert_eq!(calls.load(Ordering::Acquire), 3);
service.shutdown().await;
server.abort();
}
#[tokio::test]
async fn final_handle_shutdown_cancels_hanging_unlimited_reconnect() {
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::{Notify, oneshot};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (first_closed_tx, first_closed_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (first, _) = listener.accept().await.unwrap();
drop(first);
let _ = first_closed_tx.send(());
});
let calls = Arc::new(AtomicUsize::new(0));
let reconnect_entered = Arc::new(Notify::new());
let factory_calls = Arc::clone(&calls);
let factory_entered = Arc::clone(&reconnect_entered);
let factory = move || {
let factory_calls = Arc::clone(&factory_calls);
let factory_entered = Arc::clone(&factory_entered);
async move {
if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
factory_entered.notify_one();
return futures::future::pending::<Result<RedisConnection, RedisError>>().await;
}
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|error| RedisError::connection(addr.to_string(), error))?;
Ok(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
}
};
let events = ConnectionEventBus::new(8);
let mut event_stream = events.subscribe();
let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
max_retries: None,
base_delay: Duration::ZERO,
max_delay: Duration::from_secs(5),
jitter: false,
connect_timeout: None,
});
let service = AutoPipelineService::with_factory_and_events(
factory,
AutoPipelineConfig::default(),
reconnect,
events,
)
.await
.unwrap();
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
first_closed_rx.await.unwrap();
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt {
attempt: 1,
delay: Duration::ZERO,
}
);
reconnect_entered.notified().await;
tokio::time::timeout(Duration::from_secs(1), service.shutdown())
.await
.expect("final-handle shutdown must cancel a hanging reconnect factory");
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
assert_eq!(
event_stream.recv().await.unwrap_err(),
crate::reconnect::ConnectionEventRecvError::Closed
);
assert_eq!(calls.load(Ordering::Acquire), 2);
server.await.unwrap();
}
#[tokio::test]
async fn exhausted_worker_still_publishes_shutdown_on_final_handle() {
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::oneshot;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (first_closed_tx, first_closed_rx) = oneshot::channel();
let server = tokio::spawn(async move {
let (first, _) = listener.accept().await.unwrap();
drop(first);
let _ = first_closed_tx.send(());
});
let calls = Arc::new(AtomicUsize::new(0));
let factory_calls = Arc::clone(&calls);
let factory = move || {
let factory_calls = Arc::clone(&factory_calls);
async move {
if factory_calls.fetch_add(1, Ordering::AcqRel) > 0 {
return Err(RedisError::ConnectionClosed);
}
let stream = tokio::net::TcpStream::connect(addr)
.await
.map_err(|error| RedisError::connection(addr.to_string(), error))?;
Ok(RedisConnection::from_stream(
redis_tower_core::RedisStream::Tcp(stream),
))
}
};
let events = ConnectionEventBus::new(8);
let mut event_stream = events.subscribe();
let reconnect = AutoPipelineReconnectConfig::new(ReconnectConfig {
max_retries: Some(0),
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: false,
connect_timeout: None,
});
let service = AutoPipelineService::with_factory_and_events(
factory,
AutoPipelineConfig::default(),
reconnect,
events,
)
.await
.unwrap();
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
first_closed_rx.await.unwrap();
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectAttempt { attempt: 1, .. }
));
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectFailed { attempt: 1, .. }
));
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::ReconnectExhausted { attempts: 1 }
);
tokio::time::timeout(Duration::from_secs(1), service.shutdown())
.await
.expect("shutdown should join an already exhausted worker");
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
assert_eq!(
event_stream.recv().await.unwrap_err(),
crate::reconnect::ConnectionEventRecvError::Closed
);
assert_eq!(calls.load(Ordering::Acquire), 2);
server.await.unwrap();
}
#[test]
fn config_defaults() {
let config = AutoPipelineConfig::default();
assert_eq!(config.max_batch_size, 100);
assert_eq!(config.batch_window, Duration::ZERO);
assert_eq!(config.queue_capacity, 1024);
assert!(
!config.shed_load_on_full,
"back-pressure is the default; load shedding is opt-in"
);
assert!(
config.response_timeout.is_none(),
"no response deadline by default"
);
assert!(
config.metrics_recorder.is_none(),
"no metrics recorder by default"
);
}
#[test]
fn config_custom() {
let config = AutoPipelineConfig {
max_batch_size: 50,
batch_window: Duration::from_micros(500),
queue_capacity: 512,
shed_load_on_full: true,
response_timeout: Some(Duration::from_millis(250)),
metrics_recorder: None,
reconnect_on_readonly: false,
};
assert_eq!(config.max_batch_size, 50);
assert_eq!(config.batch_window, Duration::from_micros(500));
assert_eq!(config.queue_capacity, 512);
assert!(config.shed_load_on_full);
assert_eq!(config.response_timeout, Some(Duration::from_millis(250)));
assert!(!config.reconnect_on_readonly);
}
#[cfg(unix)]
#[tokio::test]
async fn fixed_worker_detects_idle_close_and_closes_health_and_queue() {
let (client, server) = tokio::net::UnixStream::pair().unwrap();
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
let events = ConnectionEventBus::new(4);
let mut event_stream = events.subscribe();
let mut service =
AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
let mut health = service.subscribe_connection_health();
assert!(service.is_connection_healthy());
assert!(*health.borrow());
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
drop(server);
let disconnected = tokio::time::timeout(Duration::from_secs(1), event_stream.recv())
.await
.expect("idle close was not detected")
.unwrap();
assert!(matches!(
disconnected,
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
assert!(
!service.is_connection_healthy(),
"health must be false before Disconnected is observable"
);
health
.changed()
.await
.expect("the final unhealthy snapshot must be observable");
assert!(!*health.borrow_and_update());
assert!(
health.changed().await.is_err(),
"terminal fixed-worker loss must close the health channel"
);
assert!(
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.is_err(),
"terminal idle loss must close the request queue"
);
service.shutdown().await;
}
#[cfg(unix)]
#[tokio::test]
async fn pipeline_failure_publishes_unhealthy_before_failing_responder() {
use futures::StreamExt;
use tokio_util::codec::Framed;
let (client, server) = tokio::net::UnixStream::pair().unwrap();
let server = tokio::spawn(async move {
let mut framed = Framed::new(
redis_tower_core::RedisStream::Unix(server),
redis_tower_core::RespCodec::new(),
);
framed
.next()
.await
.expect("client closed before dispatch")
.expect("client sent an invalid request");
});
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
let events = ConnectionEventBus::new(4);
let mut event_stream = events.subscribe();
let mut service =
AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
assert!(
service
.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
.await
.is_err()
);
assert!(
!service.is_connection_healthy(),
"the response future observed failure before shared health changed"
);
assert!(matches!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::ConnectionError { .. }
}
));
service.shutdown().await;
server.await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn response_timeout_publishes_unhealthy_before_failing_responder() {
use futures::StreamExt;
use tokio_util::codec::Framed;
let (client, server) = tokio::net::UnixStream::pair().unwrap();
let server = tokio::spawn(async move {
let mut framed = Framed::new(
redis_tower_core::RedisStream::Unix(server),
redis_tower_core::RespCodec::new(),
);
framed
.next()
.await
.expect("client closed before dispatch")
.expect("client sent an invalid request");
let _ = framed.next().await;
});
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
let events = ConnectionEventBus::new(4);
let mut event_stream = events.subscribe();
let config = AutoPipelineConfig {
response_timeout: Some(Duration::from_millis(10)),
..AutoPipelineConfig::default()
};
let mut service = AutoPipelineService::new_with_events(conn, config, events);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
assert!(matches!(
service
.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
.await,
Err(RedisError::CommandTimeout)
));
assert!(
!service.is_connection_healthy(),
"the timeout responder was notified before shared health changed"
);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::CommandTimeout,
}
);
service.shutdown().await;
server.await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn idle_push_is_routed_without_delaying_the_next_request() {
use futures::{SinkExt, StreamExt};
use tokio_util::codec::Framed;
let (client, server) = tokio::net::UnixStream::pair().unwrap();
let mut conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
let mut pushes = conn.subscribe_pushes();
let expected_push = Frame::Push(vec![
Frame::SimpleString(bytes::Bytes::from_static(b"invalidate")),
Frame::Array(Some(vec![Frame::SimpleString(bytes::Bytes::from_static(
b"key",
))])),
]);
let server_push = expected_push.clone();
let server = tokio::spawn(async move {
let mut framed = Framed::new(
redis_tower_core::RedisStream::Unix(server),
redis_tower_core::RespCodec::new(),
);
framed.send(server_push).await.unwrap();
framed
.next()
.await
.expect("client closed before request")
.expect("client sent an invalid request");
framed
.send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
.await
.unwrap();
let _ = framed.next().await;
});
let mut service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
assert_eq!(
tokio::time::timeout(Duration::from_secs(1), pushes.recv())
.await
.expect("idle worker did not route the push")
.unwrap(),
expected_push
);
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
assert_eq!(
service
.call(Frame::SimpleString(bytes::Bytes::from_static(b"PING")))
.await
.unwrap(),
Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
);
assert!(service.is_connection_healthy());
service.shutdown().await;
server.await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn cancelled_single_and_multi_requests_are_pruned_before_wire() {
use tokio::io::AsyncReadExt;
let (client, mut server) = tokio::net::UnixStream::pair().unwrap();
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Unix(client));
let batch_window = Duration::from_millis(200);
let mut service = AutoPipelineService::new(
conn,
AutoPipelineConfig {
batch_window,
..AutoPipelineConfig::default()
},
);
futures::future::poll_fn(|cx| service.poll_ready(cx))
.await
.unwrap();
let single = service.call(Frame::SimpleString(bytes::Bytes::from_static(
b"CANCELLED-SINGLE",
)));
assert!(
tokio::time::timeout(Duration::from_millis(25), single)
.await
.is_err(),
"single request unexpectedly completed without a server response"
);
let multi = service.call_pipeline(vec![
Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-1")),
Frame::SimpleString(bytes::Bytes::from_static(b"CANCELLED-MULTI-2")),
]);
assert!(
tokio::time::timeout(Duration::from_millis(25), multi)
.await
.is_err(),
"multi request unexpectedly completed without a server response"
);
let mut bytes = [0u8; 256];
assert!(
tokio::time::timeout(
batch_window + Duration::from_millis(100),
server.read(&mut bytes)
)
.await
.is_err(),
"a cancelled queued request reached the Redis socket"
);
service.shutdown().await;
}
#[test]
fn is_readonly_frame_detects_readonly_errors_only() {
use bytes::Bytes;
assert!(is_readonly_frame(&Frame::Error(Bytes::from(
"READONLY You can't write against a read only replica."
))));
assert!(is_readonly_frame(&Frame::Error(Bytes::from(
"readonly nope"
))));
assert!(!is_readonly_frame(&Frame::Error(Bytes::from(
"WRONGTYPE Operation against a key"
))));
assert!(!is_readonly_frame(&Frame::Error(Bytes::from("READ"))));
assert!(!is_readonly_frame(&Frame::SimpleString(Bytes::from("OK"))));
}
fn make_test_svc(
tx: mpsc::Sender<WorkerRequest>,
handle: tokio::task::JoinHandle<()>,
shed_load: bool,
) -> AutoPipelineService {
let (shutdown_tx, _shutdown_rx) = watch::channel(false);
let (_connection_health_tx, connection_health) = watch::channel(true);
let control = Arc::new(WorkerControl::new(shutdown_tx, None));
AutoPipelineService {
poll_tx: PollSender::new(tx.clone()),
tx,
shed_load,
lease: WorkerLease { control },
connection_health,
worker: Arc::new(WorkerHandle::new(handle)),
}
}
#[tokio::test]
async fn closed_channel_error_is_retryable() {
let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
drop(rx);
let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
let frame = Frame::SimpleString(b"PING"[..].into());
let err = svc.call(frame).await.unwrap_err();
assert!(err.is_retryable());
}
#[tokio::test]
async fn closed_channel_returns_error() {
let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
drop(rx); let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
let ready = futures::future::poll_fn(|cx| svc.poll_ready(cx)).await;
assert!(ready.is_err());
let frame = Frame::SimpleString(b"PING"[..].into());
let result = svc.call(frame).await;
assert!(result.is_err());
}
#[tokio::test]
async fn shutdown_drains_an_unpolled_shed_load_call() {
use futures::{SinkExt, StreamExt};
use tokio_util::codec::Framed;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let request = Frame::Array(Some(vec![Frame::BulkString(Some(
bytes::Bytes::from_static(b"PING"),
))]));
let expected_request = request.clone();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut framed = Framed::new(
redis_tower_core::RedisStream::Tcp(stream),
redis_tower_core::RespCodec::new(),
);
assert_eq!(framed.next().await.unwrap().unwrap(), expected_request);
framed
.send(Frame::SimpleString(bytes::Bytes::from_static(b"PONG")))
.await
.unwrap();
});
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
let events = ConnectionEventBus::new(4);
let mut event_stream = events.subscribe();
let config = AutoPipelineConfig {
batch_window: Duration::from_secs(30),
shed_load_on_full: true,
..AutoPipelineConfig::default()
};
let mut service = AutoPipelineService::new_with_events(conn, config, events);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
let response = service.call(request);
tokio::time::timeout(Duration::from_secs(1), service.shutdown())
.await
.expect("an unpolled shed-load call must not keep shutdown alive");
assert_eq!(
response.await.unwrap(),
Frame::SimpleString(bytes::Bytes::from_static(b"PONG"))
);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
assert_eq!(
event_stream.recv().await.unwrap_err(),
crate::reconnect::ConnectionEventRecvError::Closed
);
server.await.unwrap();
}
#[tokio::test]
async fn final_shutdown_closes_an_idle_worker_with_a_retained_sender() {
use tokio::io::AsyncReadExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let mut byte = [0_u8; 1];
assert_eq!(stream.read(&mut byte).await.unwrap(), 0);
});
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
let service = AutoPipelineService::new(conn, AutoPipelineConfig::default());
let retained_sender = service.tx.clone();
tokio::time::timeout(Duration::from_secs(1), service.shutdown())
.await
.expect("final shutdown must wake an idle worker");
assert!(retained_sender.is_closed());
drop(retained_sender);
tokio::time::timeout(Duration::from_secs(1), server)
.await
.expect("worker shutdown must close its Redis connection")
.unwrap();
}
#[tokio::test]
async fn shutdown_last_clone_awaits_worker() {
let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
let handle = tokio::spawn(async move {
let mut rx = rx;
while rx.recv().await.is_some() {}
});
let svc = make_test_svc(tx, handle, false);
svc.shutdown().await;
}
#[tokio::test]
async fn shutdown_publishes_one_intentional_disconnect() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let _stream = stream;
futures::future::pending::<()>().await;
});
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
let conn = RedisConnection::from_stream(redis_tower_core::RedisStream::Tcp(stream));
let events = ConnectionEventBus::new(4);
let mut event_stream = events.subscribe();
let service =
AutoPipelineService::new_with_events(conn, AutoPipelineConfig::default(), events);
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Connected
);
service.shutdown().await;
assert_eq!(
event_stream.recv().await.unwrap(),
ConnectionEvent::Disconnected {
reason: ConnectionDisconnectReason::Shutdown,
}
);
server.abort();
}
#[tokio::test]
async fn shutdown_non_last_clone_returns_immediately() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
let handle = tokio::spawn(futures::future::pending::<()>());
let svc = make_test_svc(tx, handle, false);
let _clone = svc.clone();
svc.shutdown().await;
}
#[tokio::test]
async fn is_alive_tracks_worker_exit() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
let alive = make_test_svc(tx, tokio::spawn(futures::future::pending::<()>()), false);
assert!(alive.is_alive());
let (tx2, _rx2) = mpsc::channel::<WorkerRequest>(1);
let dead = make_test_svc(tx2, tokio::spawn(async {}), false);
for _ in 0..10 {
if !dead.is_alive() {
break;
}
tokio::task::yield_now().await;
}
assert!(!dead.is_alive());
}
#[tokio::test]
async fn queue_full_returns_queue_full_error() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
let (dummy_tx, _dummy_rx) = oneshot::channel();
tx.try_send(WorkerRequest::Single {
frame: Frame::SimpleString(b"PING"[..].into()),
response_tx: dummy_tx,
})
.unwrap();
let mut svc = make_test_svc(tx, tokio::spawn(async {}), true);
let frame = Frame::SimpleString(b"PING"[..].into());
let err = svc.call(frame).await.unwrap_err();
assert!(
matches!(err, RedisError::QueueFull),
"expected QueueFull, got {err:?}"
);
}
#[tokio::test]
async fn backpressure_poll_ready_pends_when_full() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
let (dummy_tx, _dummy_rx) = oneshot::channel();
tx.try_send(WorkerRequest::Single {
frame: Frame::SimpleString(b"PING"[..].into()),
response_tx: dummy_tx,
})
.unwrap();
let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
let pending = std::future::poll_fn(|cx| Poll::Ready(svc.poll_ready(cx).is_pending())).await;
assert!(
pending,
"poll_ready must pend (back-pressure) when the queue is full"
);
}
#[tokio::test]
async fn reserved_pipeline_consumes_poll_ready_slot_as_one_request() {
let (tx, mut rx) = mpsc::channel::<WorkerRequest>(1);
let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
futures::future::poll_fn(|cx| svc.poll_ready(cx))
.await
.unwrap();
let frames = vec![
Frame::SimpleString(b"CLIENT CACHING YES"[..].into()),
Frame::SimpleString(b"GET key"[..].into()),
];
let response = svc.call_reserved_pipeline(frames.clone());
let request = rx.recv().await.expect("reserved request should be queued");
match request {
WorkerRequest::Multi {
frames: actual,
response_tx,
} => {
assert_eq!(actual, frames);
response_tx
.send(Ok(vec![
Frame::SimpleString(b"OK"[..].into()),
Frame::BulkString(Some(b"value"[..].into())),
]))
.unwrap();
}
WorkerRequest::Single { .. } => panic!("reserved pipeline was split into a single"),
}
assert_eq!(response.await.unwrap().len(), 2);
}
#[tokio::test]
async fn local_response_can_release_poll_ready_reservation() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(1);
let mut svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
assert_eq!(tx.capacity(), 1);
futures::future::poll_fn(|cx| svc.poll_ready(cx))
.await
.unwrap();
assert_eq!(tx.capacity(), 0, "poll_ready should reserve capacity");
svc.release_reservation();
assert_eq!(
tx.capacity(),
1,
"a locally satisfied request must restore queue capacity"
);
}
#[tokio::test]
async fn backpressure_poll_ready_errors_when_closed() {
let (tx, rx) = mpsc::channel::<WorkerRequest>(1);
drop(rx);
let mut svc = make_test_svc(tx, tokio::spawn(async {}), false);
let ready = std::future::poll_fn(|cx| svc.poll_ready(cx)).await;
let err = ready.unwrap_err();
assert!(
err.is_retryable(),
"closed-channel error should be retryable"
);
}
#[tokio::test]
async fn queue_full_not_retryable() {
assert!(!RedisError::QueueFull.is_retryable());
}
#[tokio::test]
async fn queue_full_not_connection_error() {
assert!(!RedisError::QueueFull.is_connection_error());
}
#[test]
fn config_queue_capacity_default() {
let config = AutoPipelineConfig::default();
assert_eq!(config.queue_capacity, 1024);
}
#[tokio::test]
async fn queue_depth_zero_when_empty() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(64);
let svc = make_test_svc(tx, tokio::spawn(async {}), false);
assert_eq!(svc.queue_depth(), 0);
}
#[tokio::test]
async fn queue_depth_increases_with_pending_requests() {
let (tx, _rx) = mpsc::channel::<WorkerRequest>(10);
let svc = make_test_svc(tx.clone(), tokio::spawn(async {}), false);
assert_eq!(svc.queue_depth(), 0);
let (dummy_tx, _dummy_rx) = oneshot::channel();
tx.try_send(WorkerRequest::Single {
frame: Frame::SimpleString(b"PING"[..].into()),
response_tx: dummy_tx,
})
.unwrap();
assert_eq!(svc.queue_depth(), 1);
}
}