1use std::fmt;
36use std::future::Future;
37use std::pin::Pin;
38use std::task::{Context, Poll};
39use std::time::Duration;
40
41use redis_tower_commands::Ping;
42use redis_tower_core::{Command, Frame, RedisError};
43use tower_layer::Layer;
44use tower_resilience_circuitbreaker::{
45 CircuitBreaker as UpstreamCircuitBreaker, CircuitBreakerError,
46 CircuitBreakerHandle as UpstreamCircuitBreakerHandle,
47 CircuitBreakerLayer as UpstreamCircuitBreakerLayer, FnClassifier,
48};
49use tower_service::Service;
50
51use crate::executor::RedisExecutor;
52use crate::retry::{RetryClient, RetryPolicy};
53
54pub use tower_resilience_circuitbreaker::CircuitMetrics as RedisCircuitMetrics;
56pub use tower_resilience_circuitbreaker::CircuitState as RedisCircuitState;
58
59type ClassifierFn = fn(&Result<Frame, RedisError>) -> bool;
60type RedisClassifier = FnClassifier<ClassifierFn>;
61type InnerLayer = UpstreamCircuitBreakerLayer<RedisClassifier>;
62type InnerHandle = UpstreamCircuitBreakerHandle<RedisClassifier>;
63
64#[derive(Debug)]
65struct BreakerRequest {
66 frame: Frame,
67 deadline: Option<tokio::time::Instant>,
68}
69
70#[derive(Clone)]
73struct DeadlineFrameService<S> {
74 inner: S,
75}
76
77impl<S> DeadlineFrameService<S> {
78 fn new(inner: S) -> Self {
79 Self { inner }
80 }
81}
82
83impl<S> Service<BreakerRequest> for DeadlineFrameService<S>
84where
85 S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
86 S::Future: Send + 'static,
87{
88 type Response = Frame;
89 type Error = RedisError;
90 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
91
92 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
93 self.inner.poll_ready(cx)
94 }
95
96 fn call(&mut self, request: BreakerRequest) -> Self::Future {
97 if request
98 .deadline
99 .is_some_and(|deadline| deadline <= tokio::time::Instant::now())
100 {
101 return Box::pin(async { Err(RedisError::CommandTimeout) });
102 }
103
104 let future = self.inner.call(request.frame);
105 Box::pin(async move {
106 match request.deadline {
107 Some(deadline) => tokio::time::timeout_at(deadline, future)
108 .await
109 .map_err(|_elapsed| RedisError::CommandTimeout)?,
110 None => future.await,
111 }
112 })
113 }
114}
115
116#[derive(Clone, Debug)]
122pub struct RedisCircuitBreakerConfig {
123 pub failure_threshold: u32,
125 pub recovery_probe_interval: Duration,
127}
128
129impl Default for RedisCircuitBreakerConfig {
130 fn default() -> Self {
131 Self {
132 failure_threshold: 5,
133 recovery_probe_interval: Duration::from_secs(5),
134 }
135 }
136}
137
138pub fn redis_error_is_circuit_failure(error: &RedisError) -> bool {
143 error.is_connection_error()
144 || matches!(
145 error,
146 RedisError::ConnectTimeout | RedisError::CommandTimeout
147 )
148 || matches!(
149 error,
150 RedisError::ReconnectFailed { last_error, .. }
151 if redis_error_is_circuit_failure(last_error)
152 )
153}
154
155fn classify_redis_result(result: &Result<Frame, RedisError>) -> bool {
156 matches!(result, Err(error) if redis_error_is_circuit_failure(error))
157}
158
159#[derive(Clone)]
161pub struct RedisCircuitBreakerHandle {
162 inner: InnerHandle,
163}
164
165impl RedisCircuitBreakerHandle {
166 pub fn state(&self) -> RedisCircuitState {
168 self.inner.state()
169 }
170
171 pub fn is_open(&self) -> bool {
173 self.inner.is_open()
174 }
175
176 pub fn health_status(&self) -> &'static str {
178 self.inner.health_status()
179 }
180
181 pub fn http_status(&self) -> u16 {
183 self.inner.http_status()
184 }
185
186 pub async fn metrics(&self) -> RedisCircuitMetrics {
188 self.inner.metrics().await
189 }
190}
191
192impl fmt::Debug for RedisCircuitBreakerHandle {
193 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
194 formatter
195 .debug_struct("RedisCircuitBreakerHandle")
196 .field("state", &self.state())
197 .finish()
198 }
199}
200
201#[derive(Clone)]
206pub struct RedisCircuitBreakerLayer {
207 inner: InnerLayer,
208 handle: RedisCircuitBreakerHandle,
209 config: RedisCircuitBreakerConfig,
210}
211
212impl RedisCircuitBreakerLayer {
213 pub fn new(config: RedisCircuitBreakerConfig) -> Self {
215 let threshold = config.failure_threshold.max(1) as usize;
216 let (inner, handle) = UpstreamCircuitBreakerLayer::builder()
217 .consecutive_failures(threshold)
218 .wait_duration_in_open(config.recovery_probe_interval)
219 .permitted_calls_in_half_open(1)
220 .name("redis")
221 .failure_classifier(classify_redis_result as ClassifierFn)
222 .on_state_transition(|from, to| match to {
223 RedisCircuitState::Open => {
224 tracing::warn!(?from, ?to, "redis circuit breaker state transition")
225 }
226 _ => tracing::info!(?from, ?to, "redis circuit breaker state transition"),
227 })
228 .build_with_handle();
229 Self {
230 inner,
231 handle: RedisCircuitBreakerHandle { inner: handle },
232 config,
233 }
234 }
235
236 pub fn handle(&self) -> RedisCircuitBreakerHandle {
238 self.handle.clone()
239 }
240}
241
242impl fmt::Debug for RedisCircuitBreakerLayer {
243 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
244 formatter
245 .debug_struct("RedisCircuitBreakerLayer")
246 .field("config", &self.config)
247 .field("state", &self.handle.state())
248 .finish()
249 }
250}
251
252impl<S> Layer<S> for RedisCircuitBreakerLayer {
253 type Service = RedisCircuitBreakerService<S>;
254
255 fn layer(&self, service: S) -> Self::Service {
256 RedisCircuitBreakerService {
257 inner: self.inner.layer(DeadlineFrameService::new(service)),
258 handle: self.handle.clone(),
259 }
260 }
261}
262
263pub struct RedisCircuitBreakerService<S> {
265 inner: UpstreamCircuitBreaker<DeadlineFrameService<S>, RedisClassifier>,
266 handle: RedisCircuitBreakerHandle,
267}
268
269impl<S: Clone> Clone for RedisCircuitBreakerService<S> {
270 fn clone(&self) -> Self {
271 Self {
272 inner: self.inner.clone(),
273 handle: self.handle.clone(),
274 }
275 }
276}
277
278impl<S> RedisCircuitBreakerService<S> {
279 pub fn handle(&self) -> RedisCircuitBreakerHandle {
281 self.handle.clone()
282 }
283
284 fn call_with_deadline(
285 &mut self,
286 frame: Frame,
287 deadline: Option<tokio::time::Instant>,
288 ) -> Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>
289 where
290 S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
291 S::Future: Send + 'static,
292 {
293 let future = self.inner.call(BreakerRequest { frame, deadline });
294 Box::pin(async move { future.await.map_err(map_circuit_error) })
295 }
296}
297
298impl<S> fmt::Debug for RedisCircuitBreakerService<S> {
299 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
300 formatter
301 .debug_struct("RedisCircuitBreakerService")
302 .field("state", &self.handle.state())
303 .finish_non_exhaustive()
304 }
305}
306
307fn map_circuit_error(error: CircuitBreakerError<RedisError>) -> RedisError {
308 match error {
309 CircuitBreakerError::OpenCircuit => RedisError::CircuitOpen,
310 CircuitBreakerError::Inner(error) => error,
311 }
312}
313
314impl<S> Service<Frame> for RedisCircuitBreakerService<S>
315where
316 S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
317 S::Future: Send + 'static,
318{
319 type Response = Frame;
320 type Error = RedisError;
321 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
322
323 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
324 self.inner.poll_ready(cx).map_err(map_circuit_error)
325 }
326
327 fn call(&mut self, request: Frame) -> Self::Future {
328 self.call_with_deadline(request, None)
329 }
330}
331
332#[derive(Clone)]
337pub struct RedisCircuitBreakerClient<S> {
338 inner: RedisCircuitBreakerService<S>,
339 handle: RedisCircuitBreakerHandle,
340}
341
342impl<S> RedisCircuitBreakerClient<S> {
343 pub fn new(service: S, config: RedisCircuitBreakerConfig) -> Self {
345 let layer = RedisCircuitBreakerLayer::new(config);
346 let handle = layer.handle();
347 Self {
348 inner: layer.layer(service),
349 handle,
350 }
351 }
352
353 pub fn circuit_breaker_handle(&self) -> RedisCircuitBreakerHandle {
355 self.handle.clone()
356 }
357}
358
359impl<S> RedisCircuitBreakerClient<S>
360where
361 S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
362 S::Future: Send + 'static,
363{
364 pub fn execute<Cmd: Command + 'static>(
369 &self,
370 command: Cmd,
371 ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
372 let deadline = command.deadline();
373 let mut service = self.inner.clone();
374 async move {
375 let readiness = std::future::poll_fn(|cx| service.poll_ready(cx));
376 match deadline {
377 Some(deadline) => tokio::time::timeout_at(deadline, readiness)
378 .await
379 .map_err(|_elapsed| RedisError::CommandTimeout)?,
380 None => readiness.await,
381 }?;
382
383 if deadline.is_some_and(|deadline| deadline <= tokio::time::Instant::now()) {
384 return Err(RedisError::CommandTimeout);
385 }
386
387 let frame = command.to_frame();
388 let response = service.call_with_deadline(frame, deadline).await?;
389 if let Frame::Error(ref error) = response {
390 return Err(RedisError::Redis(
391 String::from_utf8_lossy(error).into_owned(),
392 ));
393 }
394 command.parse_response(response)
395 }
396 }
397
398 pub async fn health_check(&self) -> Result<(), RedisError> {
400 self.execute(Ping::new()).await?;
401 Ok(())
402 }
403
404 pub fn retry(&self, policy: RetryPolicy) -> RetryClient<Self> {
406 RetryClient::new(self.clone(), policy)
407 }
408}
409
410impl<S> RedisExecutor for RedisCircuitBreakerClient<S>
411where
412 S: Service<Frame, Response = Frame, Error = RedisError> + Clone + Send + 'static,
413 S::Future: Send + 'static,
414{
415 fn execute<Cmd: Command>(
416 &mut self,
417 command: Cmd,
418 ) -> impl Future<Output = Result<Cmd::Response, RedisError>> + Send {
419 RedisCircuitBreakerClient::execute(self, command)
420 }
421}
422
423#[deprecated(
425 since = "0.1.0",
426 note = "use RedisCircuitBreakerConfig; this alias will be removed in a future release"
427)]
428pub type CircuitBreakerConfig = RedisCircuitBreakerConfig;
429
430#[deprecated(
432 since = "0.1.0",
433 note = "use RedisCircuitBreakerLayer; this alias will be removed in a future release"
434)]
435pub type CircuitBreakerLayer = RedisCircuitBreakerLayer;
436
437#[deprecated(
439 since = "0.1.0",
440 note = "use RedisCircuitBreakerService; this alias will be removed in a future release"
441)]
442pub type CircuitBreakerService<S> = RedisCircuitBreakerService<S>;
443
444#[cfg(test)]
445mod tests {
446 use super::*;
447 use redis_tower_core::WithDeadline;
448 use std::sync::Arc;
449 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
450
451 #[derive(Clone)]
452 struct ToggleService {
453 fail: Arc<AtomicBool>,
454 calls: Arc<AtomicUsize>,
455 user_error: bool,
456 }
457
458 impl Service<Frame> for ToggleService {
459 type Response = Frame;
460 type Error = RedisError;
461 type Future = std::future::Ready<Result<Frame, RedisError>>;
462
463 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
464 Poll::Ready(Ok(()))
465 }
466
467 fn call(&mut self, _request: Frame) -> Self::Future {
468 self.calls.fetch_add(1, Ordering::SeqCst);
469 if self.user_error {
470 std::future::ready(Err(RedisError::Redis(
471 "WRONGTYPE Operation against a key".into(),
472 )))
473 } else if self.fail.load(Ordering::SeqCst) {
474 std::future::ready(Err(RedisError::ConnectionClosed))
475 } else {
476 std::future::ready(Ok(Frame::SimpleString("PONG".into())))
477 }
478 }
479 }
480
481 #[derive(Clone)]
482 struct NeverReadyService {
483 calls: Arc<AtomicUsize>,
484 }
485
486 impl Service<Frame> for NeverReadyService {
487 type Response = Frame;
488 type Error = RedisError;
489 type Future = std::future::Ready<Result<Frame, RedisError>>;
490
491 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
492 Poll::Pending
493 }
494
495 fn call(&mut self, _request: Frame) -> Self::Future {
496 self.calls.fetch_add(1, Ordering::Relaxed);
497 std::future::ready(Ok(Frame::SimpleString("PONG".into())))
498 }
499 }
500
501 #[derive(Clone)]
502 struct SlowService {
503 calls: Arc<AtomicUsize>,
504 delay: Duration,
505 }
506
507 impl Service<Frame> for SlowService {
508 type Response = Frame;
509 type Error = RedisError;
510 type Future = Pin<Box<dyn Future<Output = Result<Frame, RedisError>> + Send>>;
511
512 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
513 Poll::Ready(Ok(()))
514 }
515
516 fn call(&mut self, _request: Frame) -> Self::Future {
517 self.calls.fetch_add(1, Ordering::Relaxed);
518 let delay = self.delay;
519 Box::pin(async move {
520 tokio::time::sleep(delay).await;
521 Ok(Frame::SimpleString("PONG".into()))
522 })
523 }
524 }
525
526 fn ping() -> Frame {
527 Frame::Array(Some(vec![Frame::BulkString(Some("PING".into()))]))
528 }
529
530 #[tokio::test]
531 async fn typed_deadline_bounds_circuit_breaker_readiness() {
532 let calls = Arc::new(AtomicUsize::new(0));
533 let config = RedisCircuitBreakerConfig {
534 failure_threshold: 1,
535 recovery_probe_interval: Duration::from_secs(5),
536 };
537 let client = RedisCircuitBreakerClient::new(
538 NeverReadyService {
539 calls: Arc::clone(&calls),
540 },
541 config,
542 );
543 let handle = client.circuit_breaker_handle();
544
545 let result = client
546 .execute(WithDeadline::after(Ping::new(), Duration::from_millis(20)))
547 .await;
548
549 assert!(matches!(result, Err(RedisError::CommandTimeout)));
550 assert_eq!(calls.load(Ordering::Relaxed), 0);
551 assert_eq!(handle.state(), RedisCircuitState::Closed);
552 assert_eq!(handle.metrics().await.total_calls, 0);
553 }
554
555 #[tokio::test]
556 async fn dispatched_typed_deadline_counts_as_circuit_failure() {
557 let calls = Arc::new(AtomicUsize::new(0));
558 let client = RedisCircuitBreakerClient::new(
559 SlowService {
560 calls: Arc::clone(&calls),
561 delay: Duration::from_millis(100),
562 },
563 RedisCircuitBreakerConfig {
564 failure_threshold: 1,
565 recovery_probe_interval: Duration::from_secs(5),
566 },
567 );
568 let handle = client.circuit_breaker_handle();
569
570 let result = client
571 .execute(WithDeadline::after(Ping::new(), Duration::from_millis(20)))
572 .await;
573
574 assert!(matches!(result, Err(RedisError::CommandTimeout)));
575 assert_eq!(calls.load(Ordering::Relaxed), 1);
576 assert_eq!(handle.state(), RedisCircuitState::Open);
577 }
578
579 #[tokio::test]
580 async fn infrastructure_failure_opens_and_maps_rejection() {
581 let calls = Arc::new(AtomicUsize::new(0));
582 let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
583 failure_threshold: 1,
584 recovery_probe_interval: Duration::from_secs(5),
585 });
586 let handle = layer.handle();
587 let mut service = layer.layer(ToggleService {
588 fail: Arc::new(AtomicBool::new(true)),
589 calls: Arc::clone(&calls),
590 user_error: false,
591 });
592
593 assert!(matches!(
594 service.call(ping()).await,
595 Err(RedisError::ConnectionClosed)
596 ));
597 assert_eq!(handle.state(), RedisCircuitState::Open);
598 assert!(matches!(
599 service.call(ping()).await,
600 Err(RedisError::CircuitOpen)
601 ));
602 assert_eq!(calls.load(Ordering::SeqCst), 1);
603 }
604
605 #[tokio::test]
606 async fn user_error_does_not_degrade_circuit_health() {
607 let calls = Arc::new(AtomicUsize::new(0));
608 let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
609 failure_threshold: 1,
610 recovery_probe_interval: Duration::from_secs(5),
611 });
612 let handle = layer.handle();
613 let mut service = layer.layer(ToggleService {
614 fail: Arc::new(AtomicBool::new(false)),
615 calls: Arc::clone(&calls),
616 user_error: true,
617 });
618
619 for _ in 0..3 {
620 assert!(matches!(
621 service.call(ping()).await,
622 Err(RedisError::Redis(message)) if message.starts_with("WRONGTYPE")
623 ));
624 }
625 assert_eq!(handle.state(), RedisCircuitState::Closed);
626 assert_eq!(calls.load(Ordering::SeqCst), 3);
627 }
628
629 #[tokio::test]
630 async fn successful_half_open_probe_closes_circuit() {
631 let fail = Arc::new(AtomicBool::new(true));
632 let layer = RedisCircuitBreakerLayer::new(RedisCircuitBreakerConfig {
633 failure_threshold: 1,
634 recovery_probe_interval: Duration::from_millis(20),
635 });
636 let handle = layer.handle();
637 let mut service = layer.layer(ToggleService {
638 fail: Arc::clone(&fail),
639 calls: Arc::new(AtomicUsize::new(0)),
640 user_error: false,
641 });
642
643 let _ = service.call(ping()).await;
644 assert_eq!(handle.state(), RedisCircuitState::Open);
645
646 fail.store(false, Ordering::SeqCst);
647 tokio::time::sleep(Duration::from_millis(30)).await;
648 assert!(service.call(ping()).await.is_ok());
649 assert_eq!(handle.state(), RedisCircuitState::Closed);
650 }
651
652 #[test]
653 fn classifier_counts_only_infrastructure_failures() {
654 assert!(redis_error_is_circuit_failure(
655 &RedisError::ConnectionClosed
656 ));
657 assert!(redis_error_is_circuit_failure(&RedisError::ConnectTimeout));
658 assert!(redis_error_is_circuit_failure(&RedisError::CommandTimeout));
659 assert!(redis_error_is_circuit_failure(
660 &RedisError::ReconnectFailed {
661 attempts: 1,
662 last_error: Arc::new(RedisError::ConnectTimeout),
663 }
664 ));
665 assert!(!redis_error_is_circuit_failure(&RedisError::Redis(
666 "WRONGTYPE bad value".into()
667 )));
668 assert!(!redis_error_is_circuit_failure(&RedisError::Redis(
669 "MOVED 1 127.0.0.1:6379".into()
670 )));
671 }
672}