Skip to main content

redis_tower/
circuit_breaker.rs

1//! Redis-aware adapter for `tower-resilience` circuit breaking.
2//!
3//! The adapter delegates state management, half-open admission, metrics, and
4//! event handling to `tower-resilience-circuitbreaker`. It preserves
5//! `redis-tower`'s public `RedisError` surface by mapping an upstream open
6//! rejection to [`RedisError::CircuitOpen`] and passing inner errors through.
7//!
8//! By default only infrastructure failures count toward opening the circuit:
9//! connection failures, connect timeouts, and command timeouts. Redis command
10//! errors such as `WRONGTYPE`, `NOSCRIPT`, `MOVED`, and `ASK` are returned to
11//! the caller without degrading circuit health.
12//!
13//! # Example
14//!
15//! ```no_run
16//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
17//! use std::time::Duration;
18//! use redis_tower::{
19//!     MultiplexedClient, RedisCircuitBreakerConfig,
20//! };
21//!
22//! let client = MultiplexedClient::connect("127.0.0.1:6379")
23//!     .await?
24//!     .with_circuit_breaker(RedisCircuitBreakerConfig {
25//!         failure_threshold: 5,
26//!         recovery_probe_interval: Duration::from_secs(5),
27//!     });
28//!
29//! let health = client.circuit_breaker_handle().health_status();
30//! assert_eq!(health, "healthy");
31//! # Ok(())
32//! # }
33//! ```
34
35use std::fmt;
36use std::future::Future;
37use std::pin::Pin;
38use std::task::{Context, Poll};
39use std::time::Duration;
40
41use redis_tower_commands::Ping;
42use redis_tower_core::{Command, Frame, RedisError};
43use tower_layer::Layer;
44use tower_resilience_circuitbreaker::{
45    CircuitBreaker as UpstreamCircuitBreaker, CircuitBreakerError,
46    CircuitBreakerHandle as UpstreamCircuitBreakerHandle,
47    CircuitBreakerLayer as UpstreamCircuitBreakerLayer, FnClassifier,
48};
49use tower_service::Service;
50
51use crate::executor::RedisExecutor;
52use crate::retry::{RetryClient, RetryPolicy};
53
54/// Metrics snapshot maintained by the upstream circuit breaker.
55pub use tower_resilience_circuitbreaker::CircuitMetrics as RedisCircuitMetrics;
56/// Observable state of a Redis circuit breaker.
57pub use tower_resilience_circuitbreaker::CircuitState as RedisCircuitState;
58
59type ClassifierFn = fn(&Result<Frame, RedisError>) -> bool;
60type RedisClassifier = FnClassifier<ClassifierFn>;
61type InnerLayer = UpstreamCircuitBreakerLayer<RedisClassifier>;
62type InnerHandle = UpstreamCircuitBreakerHandle<RedisClassifier>;
63
64#[derive(Debug)]
65struct BreakerRequest {
66    frame: Frame,
67    deadline: Option<tokio::time::Instant>,
68}
69
70/// Applies a typed command's absolute deadline inside the breaker so the
71/// breaker observes and classifies an in-flight timeout.
72#[derive(Clone)]
73struct DeadlineFrameService<S> {
74    inner: S,
75}
76
77impl<S> DeadlineFrameService<S> {
78    fn new(inner: S) -> Self {
79        Self { inner }
80    }
81}
82
83impl<S> Service<BreakerRequest> for DeadlineFrameService<S>
84where
85    S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
86    S::Future: Send + 'static,
87{
88    type Response = Frame;
89    type Error = RedisError;
90    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
91
92    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
93        self.inner.poll_ready(cx)
94    }
95
96    fn call(&mut self, request: BreakerRequest) -> Self::Future {
97        if request
98            .deadline
99            .is_some_and(|deadline| deadline <= tokio::time::Instant::now())
100        {
101            return Box::pin(async { Err(RedisError::CommandTimeout) });
102        }
103
104        let future = self.inner.call(request.frame);
105        Box::pin(async move {
106            match request.deadline {
107                Some(deadline) => tokio::time::timeout_at(deadline, future)
108                    .await
109                    .map_err(|_elapsed| RedisError::CommandTimeout)?,
110                None => future.await,
111            }
112        })
113    }
114}
115
116/// Configuration for the Redis-aware circuit breaker.
117///
118/// The settings retain the semantics of the original redis-tower breaker:
119/// `failure_threshold` is a consecutive-failure count and a successful call
120/// resets it. One call is admitted while half-open.
121#[derive(Clone, Debug)]
122pub struct RedisCircuitBreakerConfig {
123    /// Consecutive classified failures before the circuit opens (default: 5).
124    pub failure_threshold: u32,
125    /// How long to remain open before allowing a recovery probe (default: 5s).
126    pub recovery_probe_interval: Duration,
127}
128
129impl Default for RedisCircuitBreakerConfig {
130    fn default() -> Self {
131        Self {
132            failure_threshold: 5,
133            recovery_probe_interval: Duration::from_secs(5),
134        }
135    }
136}
137
138/// Return whether a Redis error represents an infrastructure failure.
139///
140/// Connection failures, connect timeouts, and command timeouts count toward
141/// opening the circuit. Server responses and caller errors do not.
142pub fn redis_error_is_circuit_failure(error: &RedisError) -> bool {
143    error.is_connection_error()
144        || matches!(
145            error,
146            RedisError::ConnectTimeout | RedisError::CommandTimeout
147        )
148        || matches!(
149            error,
150            RedisError::ReconnectFailed { last_error, .. }
151                if redis_error_is_circuit_failure(last_error)
152        )
153}
154
155fn classify_redis_result(result: &Result<Frame, RedisError>) -> bool {
156    matches!(result, Err(error) if redis_error_is_circuit_failure(error))
157}
158
159/// Read-only handle for circuit state and metrics.
160#[derive(Clone)]
161pub struct RedisCircuitBreakerHandle {
162    inner: InnerHandle,
163}
164
165impl RedisCircuitBreakerHandle {
166    /// Return the current circuit state without waiting for a lock.
167    pub fn state(&self) -> RedisCircuitState {
168        self.inner.state()
169    }
170
171    /// Return whether the circuit is open.
172    pub fn is_open(&self) -> bool {
173        self.inner.is_open()
174    }
175
176    /// Return `healthy`, `degraded`, or `unhealthy` for health endpoints.
177    pub fn health_status(&self) -> &'static str {
178        self.inner.health_status()
179    }
180
181    /// Return an HTTP-compatible health status (`200` or `503`).
182    pub fn http_status(&self) -> u16 {
183        self.inner.http_status()
184    }
185
186    /// Snapshot the circuit's call and failure metrics.
187    pub async fn metrics(&self) -> RedisCircuitMetrics {
188        self.inner.metrics().await
189    }
190}
191
192impl fmt::Debug for RedisCircuitBreakerHandle {
193    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
194        formatter
195            .debug_struct("RedisCircuitBreakerHandle")
196            .field("state", &self.state())
197            .finish()
198    }
199}
200
201/// Tower layer backed by `tower-resilience-circuitbreaker`.
202///
203/// All services produced by a layer share one circuit state. The inner service
204/// must be cloneable because upstream moves a clone into each call future.
205#[derive(Clone)]
206pub struct RedisCircuitBreakerLayer {
207    inner: InnerLayer,
208    handle: RedisCircuitBreakerHandle,
209    config: RedisCircuitBreakerConfig,
210}
211
212impl RedisCircuitBreakerLayer {
213    /// Build a Redis-aware breaker using consecutive-failure semantics.
214    pub fn new(config: RedisCircuitBreakerConfig) -> Self {
215        let threshold = config.failure_threshold.max(1) as usize;
216        let (inner, handle) = UpstreamCircuitBreakerLayer::builder()
217            .consecutive_failures(threshold)
218            .wait_duration_in_open(config.recovery_probe_interval)
219            .permitted_calls_in_half_open(1)
220            .name("redis")
221            .failure_classifier(classify_redis_result as ClassifierFn)
222            .on_state_transition(|from, to| match to {
223                RedisCircuitState::Open => {
224                    tracing::warn!(?from, ?to, "redis circuit breaker state transition")
225                }
226                _ => tracing::info!(?from, ?to, "redis circuit breaker state transition"),
227            })
228            .build_with_handle();
229        Self {
230            inner,
231            handle: RedisCircuitBreakerHandle { inner: handle },
232            config,
233        }
234    }
235
236    /// Clone a read-only handle for health checks and operational metrics.
237    pub fn handle(&self) -> RedisCircuitBreakerHandle {
238        self.handle.clone()
239    }
240}
241
242impl fmt::Debug for RedisCircuitBreakerLayer {
243    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
244        formatter
245            .debug_struct("RedisCircuitBreakerLayer")
246            .field("config", &self.config)
247            .field("state", &self.handle.state())
248            .finish()
249    }
250}
251
252impl<S> Layer<S> for RedisCircuitBreakerLayer {
253    type Service = RedisCircuitBreakerService<S>;
254
255    fn layer(&self, service: S) -> Self::Service {
256        RedisCircuitBreakerService {
257            inner: self.inner.layer(DeadlineFrameService::new(service)),
258            handle: self.handle.clone(),
259        }
260    }
261}
262
263/// RedisError-preserving service produced by [`RedisCircuitBreakerLayer`].
264pub struct RedisCircuitBreakerService<S> {
265    inner: UpstreamCircuitBreaker<DeadlineFrameService<S>, RedisClassifier>,
266    handle: RedisCircuitBreakerHandle,
267}
268
269impl<S: Clone> Clone for RedisCircuitBreakerService<S> {
270    fn clone(&self) -> Self {
271        Self {
272            inner: self.inner.clone(),
273            handle: self.handle.clone(),
274        }
275    }
276}
277
278impl<S> RedisCircuitBreakerService<S> {
279    /// Clone a read-only handle for health checks and operational metrics.
280    pub fn handle(&self) -> RedisCircuitBreakerHandle {
281        self.handle.clone()
282    }
283
284    fn call_with_deadline(
285        &mut self,
286        frame: Frame,
287        deadline: Option<tokio::time::Instant>,
288    ) -> Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>
289    where
290        S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
291        S::Future: Send + 'static,
292    {
293        let future = self.inner.call(BreakerRequest { frame, deadline });
294        Box::pin(async move { future.await.map_err(map_circuit_error) })
295    }
296}
297
298impl<S> fmt::Debug for RedisCircuitBreakerService<S> {
299    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
300        formatter
301            .debug_struct("RedisCircuitBreakerService")
302            .field("state", &self.handle.state())
303            .finish_non_exhaustive()
304    }
305}
306
307fn map_circuit_error(error: CircuitBreakerError<RedisError>) -> RedisError {
308    match error {
309        CircuitBreakerError::OpenCircuit => RedisError::CircuitOpen,
310        CircuitBreakerError::Inner(error) => error,
311    }
312}
313
314impl<S> Service<Frame> for RedisCircuitBreakerService<S>
315where
316    S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
317    S::Future: Send + 'static,
318{
319    type Response = Frame;
320    type Error = RedisError;
321    type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
322
323    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
324        self.inner.poll_ready(cx).map_err(map_circuit_error)
325    }
326
327    fn call(&mut self, request: Frame) -> Self::Future {
328        self.call_with_deadline(request, None)
329    }
330}
331
332/// Typed high-level client protected by a Redis circuit breaker.
333///
334/// Created by `MultiplexedClient::with_circuit_breaker` or
335/// `ResilientRedisClient::with_circuit_breaker`.
336#[derive(Clone)]
337pub struct RedisCircuitBreakerClient<S> {
338    inner: RedisCircuitBreakerService<S>,
339    handle: RedisCircuitBreakerHandle,
340}
341
342impl<S> RedisCircuitBreakerClient<S> {
343    /// Wrap a cloneable frame service in the Redis-aware circuit breaker.
344    pub fn new(service: S, config: RedisCircuitBreakerConfig) -> Self {
345        let layer = RedisCircuitBreakerLayer::new(config);
346        let handle = layer.handle();
347        Self {
348            inner: layer.layer(service),
349            handle,
350        }
351    }
352
353    /// Clone the operational handle for this client's shared circuit.
354    pub fn circuit_breaker_handle(&self) -> RedisCircuitBreakerHandle {
355        self.handle.clone()
356    }
357}
358
359impl<S> RedisCircuitBreakerClient<S>
360where
361    S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
362    S::Future: Send + 'static,
363{
364    /// Execute a typed command through the circuit breaker.
365    ///
366    /// A deadline carried by [`redis_tower_core::WithDeadline`] bounds both
367    /// waiting for inner readiness and the dispatched call.
368    pub fn execute<Cmd: Command + 'static>(
369        &self,
370        command: Cmd,
371    ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
372        let deadline = command.deadline();
373        let mut service = self.inner.clone();
374        async move {
375            let readiness = std::future::poll_fn(|cx| service.poll_ready(cx));
376            match deadline {
377                Some(deadline) => tokio::time::timeout_at(deadline, readiness)
378                    .await
379                    .map_err(|_elapsed| RedisError::CommandTimeout)?,
380                None => readiness.await,
381            }?;
382
383            if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
384                return Err(RedisError::CommandTimeout);
385            }
386
387            let frame = command.to_frame();
388            let response = service.call_with_deadline(frame, deadline).await?;
389            if let Frame::Error(ref error) = response {
390                return Err(RedisError::Redis(
391                    String::from_utf8_lossy(error).into_owned(),
392                ));
393            }
394            command.parse_response(response)
395        }
396    }
397
398    /// Send a PING through the breaker for a dependency health check.
399    pub async fn health_check(&self) -> Result<(), RedisError> {
400        self.execute(Ping::new()).await?;
401        Ok(())
402    }
403
404    /// Add idempotent-aware retries outside the circuit breaker.
405    pub fn retry(&self, policy: RetryPolicy) -> RetryClient<Self> {
406        RetryClient::new(self.clone(), policy)
407    }
408}
409
410impl<S> RedisExecutor for RedisCircuitBreakerClient<S>
411where
412    S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
413    S::Future: Send + 'static,
414{
415    fn execute<Cmd: Command>(
416        &mut self,
417        command: Cmd,
418    ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
419        RedisCircuitBreakerClient::execute(self, command)
420    }
421}
422
423/// Deprecated name for [`RedisCircuitBreakerConfig`].
424#[deprecated(
425    since = "0.1.0",
426    note = "use RedisCircuitBreakerConfig; this alias will be removed in a future release"
427)]
428pub type CircuitBreakerConfig = RedisCircuitBreakerConfig;
429
430/// Deprecated name for [`RedisCircuitBreakerLayer`].
431#[deprecated(
432    since = "0.1.0",
433    note = "use RedisCircuitBreakerLayer; this alias will be removed in a future release"
434)]
435pub type CircuitBreakerLayer = RedisCircuitBreakerLayer;
436
437/// Deprecated name for [`RedisCircuitBreakerService`].
438#[deprecated(
439    since = "0.1.0",
440    note = "use RedisCircuitBreakerService; this alias will be removed in a future release"
441)]
442pub type CircuitBreakerService<S> = RedisCircuitBreakerService<S>;
443
444#[cfg(test)]
445mod tests {
446    use super::*;
447    use redis_tower_core::WithDeadline;
448    use std::sync::Arc;
449    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
450
451    #[derive(Clone)]
452    struct ToggleService {
453        fail: Arc<AtomicBool>,
454        calls: Arc<AtomicUsize>,
455        user_error: bool,
456    }
457
458    impl Service<Frame> for ToggleService {
459        type Response = Frame;
460        type Error = RedisError;
461        type Future = std::future::Ready<Result<Frame, RedisError>>;
462
463        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
464            Poll::Ready(Ok(()))
465        }
466
467        fn call(&mut self, _request: Frame) -> Self::Future {
468            self.calls.fetch_add(1, Ordering::SeqCst);
469            if self.user_error {
470                std::future::ready(Err(RedisError::Redis(
471                    "WRONGTYPE Operation against a key".into(),
472                )))
473            } else if self.fail.load(Ordering::SeqCst) {
474                std::future::ready(Err(RedisError::ConnectionClosed))
475            } else {
476                std::future::ready(Ok(Frame::SimpleString("PONG".into())))
477            }
478        }
479    }
480
481    #[derive(Clone)]
482    struct NeverReadyService {
483        calls: Arc<AtomicUsize>,
484    }
485
486    impl Service<Frame> for NeverReadyService {
487        type Response = Frame;
488        type Error = RedisError;
489        type Future = std::future::Ready<Result<Frame, RedisError>>;
490
491        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
492            Poll::Pending
493        }
494
495        fn call(&mut self, _request: Frame) -> Self::Future {
496            self.calls.fetch_add(1, Ordering::Relaxed);
497            std::future::ready(Ok(Frame::SimpleString("PONG".into())))
498        }
499    }
500
501    #[derive(Clone)]
502    struct SlowService {
503        calls: Arc<AtomicUsize>,
504        delay: Duration,
505    }
506
507    impl Service<Frame> for SlowService {
508        type Response = Frame;
509        type Error = RedisError;
510        type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
511
512        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
513            Poll::Ready(Ok(()))
514        }
515
516        fn call(&mut self, _request: Frame) -> Self::Future {
517            self.calls.fetch_add(1, Ordering::Relaxed);
518            let delay = self.delay;
519            Box::pin(async move {
520                tokio::time::sleep(delay).await;
521                Ok(Frame::SimpleString("PONG".into()))
522            })
523        }
524    }
525
526    fn ping() -> Frame {
527        Frame::Array(Some(vec![Frame::BulkString(Some("PING".into()))]))
528    }
529
530    #[tokio::test]
531    async fn typed_deadline_bounds_circuit_breaker_readiness() {
532        let calls = Arc::new(AtomicUsize::new(0));
533        let config = RedisCircuitBreakerConfig {
534            failure_threshold: 1,
535            recovery_probe_interval: Duration::from_secs(5),
536        };
537        let client = RedisCircuitBreakerClient::new(
538            NeverReadyService {
539                calls: Arc::clone(&calls),
540            },
541            config,
542        );
543        let handle = client.circuit_breaker_handle();
544
545        let result = client
546            .execute(WithDeadline::after(Ping::new(), Duration::from_millis(20)))
547            .await;
548
549        assert!(matches!(result, Err(RedisError::CommandTimeout)));
550        assert_eq!(calls.load(Ordering::Relaxed), 0);
551        assert_eq!(handle.state(), RedisCircuitState::Closed);
552        assert_eq!(handle.metrics().await.total_calls, 0);
553    }
554
555    #[tokio::test]
556    async fn dispatched_typed_deadline_counts_as_circuit_failure() {
557        let calls = Arc::new(AtomicUsize::new(0));
558        let client = RedisCircuitBreakerClient::new(
559            SlowService {
560                calls: Arc::clone(&calls),
561                delay: Duration::from_millis(100),
562            },
563            RedisCircuitBreakerConfig {
564                failure_threshold: 1,
565                recovery_probe_interval: Duration::from_secs(5),
566            },
567        );
568        let handle = client.circuit_breaker_handle();
569
570        let result = client
571            .execute(WithDeadline::after(Ping::new(), Duration::from_millis(20)))
572            .await;
573
574        assert!(matches!(result, Err(RedisError::CommandTimeout)));
575        assert_eq!(calls.load(Ordering::Relaxed), 1);
576        assert_eq!(handle.state(), RedisCircuitState::Open);
577    }
578
579    #[tokio::test]
580    async fn infrastructure_failure_opens_and_maps_rejection() {
581        let calls = Arc::new(AtomicUsize::new(0));
582        let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
583            failure_threshold: 1,
584            recovery_probe_interval: Duration::from_secs(5),
585        });
586        let handle = layer.handle();
587        let mut service = layer.layer(ToggleService {
588            fail: Arc::new(AtomicBool::new(true)),
589            calls: Arc::clone(&calls),
590            user_error: false,
591        });
592
593        assert!(matches!(
594            service.call(ping()).await,
595            Err(RedisError::ConnectionClosed)
596        ));
597        assert_eq!(handle.state(), RedisCircuitState::Open);
598        assert!(matches!(
599            service.call(ping()).await,
600            Err(RedisError::CircuitOpen)
601        ));
602        assert_eq!(calls.load(Ordering::SeqCst), 1);
603    }
604
605    #[tokio::test]
606    async fn user_error_does_not_degrade_circuit_health() {
607        let calls = Arc::new(AtomicUsize::new(0));
608        let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
609            failure_threshold: 1,
610            recovery_probe_interval: Duration::from_secs(5),
611        });
612        let handle = layer.handle();
613        let mut service = layer.layer(ToggleService {
614            fail: Arc::new(AtomicBool::new(false)),
615            calls: Arc::clone(&calls),
616            user_error: true,
617        });
618
619        for _ in 0..3 {
620            assert!(matches!(
621                service.call(ping()).await,
622                Err(RedisError::Redis(message)) if message.starts_with("WRONGTYPE")
623            ));
624        }
625        assert_eq!(handle.state(), RedisCircuitState::Closed);
626        assert_eq!(calls.load(Ordering::SeqCst), 3);
627    }
628
629    #[tokio::test]
630    async fn successful_half_open_probe_closes_circuit() {
631        let fail = Arc::new(AtomicBool::new(true));
632        let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
633            failure_threshold: 1,
634            recovery_probe_interval: Duration::from_millis(20),
635        });
636        let handle = layer.handle();
637        let mut service = layer.layer(ToggleService {
638            fail: Arc::clone(&fail),
639            calls: Arc::new(AtomicUsize::new(0)),
640            user_error: false,
641        });
642
643        let _ = service.call(ping()).await;
644        assert_eq!(handle.state(), RedisCircuitState::Open);
645
646        fail.store(false, Ordering::SeqCst);
647        tokio::time::sleep(Duration::from_millis(30)).await;
648        assert!(service.call(ping()).await.is_ok());
649        assert_eq!(handle.state(), RedisCircuitState::Closed);
650    }
651
652    #[test]
653    fn classifier_counts_only_infrastructure_failures() {
654        assert!(redis_error_is_circuit_failure(
655            &RedisError::ConnectionClosed
656        ));
657        assert!(redis_error_is_circuit_failure(&RedisError::ConnectTimeout));
658        assert!(redis_error_is_circuit_failure(&RedisError::CommandTimeout));
659        assert!(redis_error_is_circuit_failure(
660            &RedisError::ReconnectFailed {
661                attempts: 1,
662                last_error: Arc::new(RedisError::ConnectTimeout),
663            }
664        ));
665        assert!(!redis_error_is_circuit_failure(&RedisError::Redis(
666            "WRONGTYPE bad value".into()
667        )));
668        assert!(!redis_error_is_circuit_failure(&RedisError::Redis(
669            "MOVED 1 127.0.0.1:6379".into()
670        )));
671    }
672}