1use std::collections::VecDeque;
25
26pub type NcbCircuitState = CircuitState;
33
34pub type NcbCircuitConfig = CircuitConfig;
37
38#[derive(Clone, Debug, PartialEq)]
48pub enum CircuitState {
49 Closed {
51 failure_count: u32,
53 success_count: u32,
55 },
56 Open {
59 opened_at: u64,
61 retry_after_us: u64,
63 },
64 HalfOpen {
67 probe_count: u32,
69 success_count: u32,
71 },
72}
73
74impl CircuitState {
75 pub fn label(&self) -> &'static str {
77 match self {
78 CircuitState::Closed { .. } => "Closed",
79 CircuitState::Open { .. } => "Open",
80 CircuitState::HalfOpen { .. } => "HalfOpen",
81 }
82 }
83}
84
85#[derive(Clone, Debug, PartialEq)]
91pub enum CircuitOutcome {
92 Success,
94 Failure(String),
96 Timeout,
98 Rejected,
102}
103
104#[derive(Clone, Debug)]
110pub struct CircuitConfig {
111 pub failure_threshold: u32,
114 pub success_threshold: u32,
117 pub half_open_probes: u32,
119 pub open_duration_us: u64,
121 pub timeout_us: u64,
124 pub sliding_window_size: usize,
127}
128
129impl Default for CircuitConfig {
130 fn default() -> Self {
131 Self {
132 failure_threshold: 5,
133 success_threshold: 2,
134 half_open_probes: 3,
135 open_duration_us: 30_000_000, timeout_us: 5_000_000, sliding_window_size: 20,
138 }
139 }
140}
141
142#[derive(Clone, Debug)]
148pub struct CircuitMetrics {
149 pub success_rate: f64,
151 pub failure_rate: f64,
153 pub rejection_rate: f64,
156 pub avg_response_time_us: f64,
158 pub total_requests: u64,
160 pub current_state: String,
163}
164
165#[derive(Clone, Debug, PartialEq)]
171pub enum CircuitEvent {
172 StateChanged {
174 from: String,
176 to: String,
178 at: u64,
180 },
181 ThresholdReached {
183 failures: u32,
185 },
186 RecoverySucceeded,
188 RecoveryFailed,
190}
191
192#[derive(Clone, Debug, PartialEq)]
198pub enum BreakerError {
199 CircuitOpen {
202 retry_after_us: u64,
204 },
205 MaxProbesExceeded,
208 ConfigurationError(String),
210}
211
212impl std::fmt::Display for BreakerError {
213 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
214 match self {
215 BreakerError::CircuitOpen { retry_after_us } => {
216 write!(f, "circuit open; retry after {}µs", retry_after_us)
217 }
218 BreakerError::MaxProbesExceeded => write!(f, "half-open probe quota exhausted"),
219 BreakerError::ConfigurationError(msg) => write!(f, "configuration error: {}", msg),
220 }
221 }
222}
223
224impl std::error::Error for BreakerError {}
225
226#[derive(Clone, Debug, Default)]
234struct SlidingWindow {
235 outcomes: VecDeque<bool>,
236 capacity: usize,
237 failure_count: u32,
238 success_count: u32,
239}
240
241impl SlidingWindow {
242 fn new(capacity: usize) -> Self {
243 Self {
244 outcomes: VecDeque::with_capacity(capacity),
245 capacity,
246 failure_count: 0,
247 success_count: 0,
248 }
249 }
250
251 fn push(&mut self, success: bool) {
253 if self.outcomes.len() == self.capacity {
254 if let Some(evicted) = self.outcomes.pop_front() {
255 if evicted {
256 self.success_count = self.success_count.saturating_sub(1);
257 } else {
258 self.failure_count = self.failure_count.saturating_sub(1);
259 }
260 }
261 }
262 self.outcomes.push_back(success);
263 if success {
264 self.success_count += 1;
265 } else {
266 self.failure_count += 1;
267 }
268 }
269
270 fn len(&self) -> usize {
271 self.outcomes.len()
272 }
273
274 fn failure_rate(&self) -> f64 {
275 if self.outcomes.is_empty() {
276 0.0
277 } else {
278 self.failure_count as f64 / self.outcomes.len() as f64
279 }
280 }
281
282 fn success_rate(&self) -> f64 {
283 if self.outcomes.is_empty() {
284 1.0
285 } else {
286 self.success_count as f64 / self.outcomes.len() as f64
287 }
288 }
289
290 fn reset(&mut self) {
291 self.outcomes.clear();
292 self.failure_count = 0;
293 self.success_count = 0;
294 }
295}
296
297#[derive(Clone, Debug, Default)]
302struct MetricsAccumulator {
303 total_requests: u64,
304 total_successes: u64,
305 total_failures: u64,
306 total_timeouts: u64,
307 total_rejections: u64,
308 total_response_time_us: u128,
309 response_time_samples: u64,
310}
311
312impl MetricsAccumulator {
313 fn record_outcome(&mut self, outcome: &CircuitOutcome, response_time_us: u64) {
314 self.total_requests += 1;
315 match outcome {
316 CircuitOutcome::Success => {
317 self.total_successes += 1;
318 }
319 CircuitOutcome::Failure(_) => {
320 self.total_failures += 1;
321 }
322 CircuitOutcome::Timeout => {
323 self.total_timeouts += 1;
324 self.total_failures += 1;
325 }
326 CircuitOutcome::Rejected => {
327 self.total_rejections += 1;
328 return; }
330 }
331 self.total_response_time_us += u128::from(response_time_us);
332 self.response_time_samples += 1;
333 }
334
335 fn avg_response_time_us(&self) -> f64 {
336 if self.response_time_samples == 0 {
337 0.0
338 } else {
339 self.total_response_time_us as f64 / self.response_time_samples as f64
340 }
341 }
342
343 fn rejection_rate(&self) -> f64 {
344 if self.total_requests == 0 {
345 0.0
346 } else {
347 self.total_rejections as f64 / self.total_requests as f64
348 }
349 }
350
351 fn reset(&mut self) {
352 *self = MetricsAccumulator::default();
353 }
354}
355
356#[derive(Debug)]
366pub struct CircuitCallGuard {
367 pub issued_at: u64,
369 recorded: bool,
371}
372
373impl CircuitCallGuard {
374 fn new(issued_at: u64) -> Self {
375 Self {
376 issued_at,
377 recorded: false,
378 }
379 }
380
381 pub fn mark_recorded(&mut self) {
384 self.recorded = true;
385 }
386
387 pub fn is_recorded(&self) -> bool {
389 self.recorded
390 }
391}
392
393impl Drop for CircuitCallGuard {
401 fn drop(&mut self) {
402 let _ = self.recorded;
405 }
406}
407
408#[inline]
414pub fn xorshift64(state: &mut u64) -> u64 {
415 let mut x = *state;
416 x ^= x << 13;
417 x ^= x >> 7;
418 x ^= x << 17;
419 *state = x;
420 x
421}
422
423const MAX_EVENT_HISTORY: usize = 50;
429
430#[derive(Debug)]
443pub struct NetworkCircuitBreaker {
444 config: CircuitConfig,
445 state: CircuitState,
446 window: SlidingWindow,
447 metrics: MetricsAccumulator,
448 events: VecDeque<CircuitEvent>,
449}
450
451impl NetworkCircuitBreaker {
452 pub fn new(config: CircuitConfig) -> Result<Self, BreakerError> {
457 if config.sliding_window_size == 0 {
458 return Err(BreakerError::ConfigurationError(
459 "sliding_window_size must be >= 1".to_string(),
460 ));
461 }
462 if config.failure_threshold == 0 {
463 return Err(BreakerError::ConfigurationError(
464 "failure_threshold must be >= 1".to_string(),
465 ));
466 }
467 if config.success_threshold == 0 {
468 return Err(BreakerError::ConfigurationError(
469 "success_threshold must be >= 1".to_string(),
470 ));
471 }
472 if config.half_open_probes == 0 {
473 return Err(BreakerError::ConfigurationError(
474 "half_open_probes must be >= 1".to_string(),
475 ));
476 }
477 let window = SlidingWindow::new(config.sliding_window_size);
478 Ok(Self {
479 config,
480 state: CircuitState::Closed {
481 failure_count: 0,
482 success_count: 0,
483 },
484 window,
485 metrics: MetricsAccumulator::default(),
486 events: VecDeque::with_capacity(MAX_EVENT_HISTORY + 1),
487 })
488 }
489
490 pub fn call(&mut self, current_ts: u64) -> Result<CircuitCallGuard, BreakerError> {
499 match &self.state {
500 CircuitState::Closed { .. } => Ok(CircuitCallGuard::new(current_ts)),
501 CircuitState::Open {
502 opened_at,
503 retry_after_us,
504 } => {
505 let threshold = opened_at.saturating_add(*retry_after_us);
506 if current_ts >= threshold {
507 let prev_label = self.state.label().to_string();
509 self.state = CircuitState::HalfOpen {
510 probe_count: 1,
511 success_count: 0,
512 };
513 self.push_event(CircuitEvent::StateChanged {
514 from: prev_label,
515 to: "HalfOpen".to_string(),
516 at: current_ts,
517 });
518 Ok(CircuitCallGuard::new(current_ts))
519 } else {
520 let remaining = threshold - current_ts;
521 self.metrics.total_requests += 1;
522 self.metrics.total_rejections += 1;
523 Err(BreakerError::CircuitOpen {
524 retry_after_us: remaining,
525 })
526 }
527 }
528 CircuitState::HalfOpen {
529 probe_count,
530 success_count,
531 } => {
532 let pc = *probe_count;
533 let sc = *success_count;
534 if pc < self.config.half_open_probes {
535 self.state = CircuitState::HalfOpen {
536 probe_count: pc + 1,
537 success_count: sc,
538 };
539 Ok(CircuitCallGuard::new(current_ts))
540 } else {
541 self.metrics.total_requests += 1;
542 self.metrics.total_rejections += 1;
543 Err(BreakerError::MaxProbesExceeded)
544 }
545 }
546 }
547 }
548
549 pub fn record_outcome(
556 &mut self,
557 outcome: CircuitOutcome,
558 response_time_us: u64,
559 current_ts: u64,
560 ) -> Option<CircuitEvent> {
561 self.metrics.record_outcome(&outcome, response_time_us);
562
563 match &self.state.clone() {
564 CircuitState::Closed { .. } => self.handle_outcome_closed(outcome, current_ts),
565 CircuitState::HalfOpen {
566 probe_count,
567 success_count,
568 } => {
569 let pc = *probe_count;
570 let sc = *success_count;
571 self.handle_outcome_half_open(outcome, current_ts, pc, sc)
572 }
573 CircuitState::Open { .. } => {
574 None
578 }
579 }
580 }
581
582 fn handle_outcome_closed(
587 &mut self,
588 outcome: CircuitOutcome,
589 current_ts: u64,
590 ) -> Option<CircuitEvent> {
591 let is_success = matches!(outcome, CircuitOutcome::Success);
592 self.window.push(is_success);
593
594 let failures_in_window = self.window.failure_count;
595 let successes_in_window = self.window.success_count;
596
597 let should_open = failures_in_window >= self.config.failure_threshold
599 && self.window.len() >= self.config.failure_threshold as usize;
600
601 if should_open {
602 let prev = self.state.label().to_string();
603 self.state = CircuitState::Open {
604 opened_at: current_ts,
605 retry_after_us: self.config.open_duration_us,
606 };
607 self.window.reset();
608 let threshold_event = CircuitEvent::ThresholdReached {
609 failures: failures_in_window,
610 };
611 let state_event = CircuitEvent::StateChanged {
612 from: prev,
613 to: "Open".to_string(),
614 at: current_ts,
615 };
616 self.push_event(threshold_event);
617 self.push_event(state_event.clone());
618 Some(state_event)
619 } else {
620 self.state = CircuitState::Closed {
622 failure_count: failures_in_window,
623 success_count: successes_in_window,
624 };
625 None
626 }
627 }
628
629 fn handle_outcome_half_open(
630 &mut self,
631 outcome: CircuitOutcome,
632 current_ts: u64,
633 _probe_count: u32,
634 success_count: u32,
635 ) -> Option<CircuitEvent> {
636 match outcome {
637 CircuitOutcome::Success => {
638 let new_successes = success_count + 1;
639 if new_successes >= self.config.success_threshold {
640 let prev = self.state.label().to_string();
642 self.state = CircuitState::Closed {
643 failure_count: 0,
644 success_count: 0,
645 };
646 self.window.reset();
647 let evt = CircuitEvent::RecoverySucceeded;
648 self.push_event(evt.clone());
649 let state_evt = CircuitEvent::StateChanged {
650 from: prev,
651 to: "Closed".to_string(),
652 at: current_ts,
653 };
654 self.push_event(state_evt);
655 Some(evt)
656 } else {
657 if let CircuitState::HalfOpen { probe_count, .. } = self.state {
659 self.state = CircuitState::HalfOpen {
660 probe_count,
661 success_count: new_successes,
662 };
663 }
664 None
665 }
666 }
667 CircuitOutcome::Failure(_) | CircuitOutcome::Timeout => {
668 let prev = self.state.label().to_string();
670 self.state = CircuitState::Open {
671 opened_at: current_ts,
672 retry_after_us: self.config.open_duration_us,
673 };
674 self.window.reset();
675 let evt = CircuitEvent::RecoveryFailed;
676 self.push_event(evt.clone());
677 let state_evt = CircuitEvent::StateChanged {
678 from: prev,
679 to: "Open".to_string(),
680 at: current_ts,
681 };
682 self.push_event(state_evt);
683 Some(evt)
684 }
685 CircuitOutcome::Rejected => None,
686 }
687 }
688
689 pub fn force_open(&mut self, current_ts: u64) {
695 let prev = self.state.label().to_string();
696 self.state = CircuitState::Open {
697 opened_at: current_ts,
698 retry_after_us: self.config.open_duration_us,
699 };
700 self.window.reset();
701 self.push_event(CircuitEvent::StateChanged {
702 from: prev,
703 to: "Open".to_string(),
704 at: current_ts,
705 });
706 }
707
708 pub fn force_close(&mut self) {
710 let prev = self.state.label().to_string();
711 self.state = CircuitState::Closed {
712 failure_count: 0,
713 success_count: 0,
714 };
715 self.window.reset();
716 self.push_event(CircuitEvent::StateChanged {
717 from: prev,
718 to: "Closed".to_string(),
719 at: 0,
720 });
721 }
722
723 pub fn state(&self) -> &CircuitState {
729 &self.state
730 }
731
732 pub fn reset_metrics(&mut self) {
736 self.metrics.reset();
737 }
738
739 pub fn metrics(&self, _current_ts: u64) -> CircuitMetrics {
741 let window_success_rate = self.window.success_rate();
742 let window_failure_rate = self.window.failure_rate();
743 CircuitMetrics {
744 success_rate: window_success_rate,
745 failure_rate: window_failure_rate,
746 rejection_rate: self.metrics.rejection_rate(),
747 avg_response_time_us: self.metrics.avg_response_time_us(),
748 total_requests: self.metrics.total_requests,
749 current_state: self.state.label().to_string(),
750 }
751 }
752
753 pub fn event_history(&self) -> Vec<CircuitEvent> {
755 self.events.iter().cloned().collect()
756 }
757
758 fn push_event(&mut self, event: CircuitEvent) {
763 if self.events.len() >= MAX_EVENT_HISTORY {
764 self.events.pop_front();
765 }
766 self.events.push_back(event);
767 }
768}
769
770#[cfg(test)]
775mod tests {
776 use super::*;
777
778 fn make_breaker() -> NetworkCircuitBreaker {
782 let cfg = CircuitConfig {
783 failure_threshold: 3,
784 success_threshold: 2,
785 half_open_probes: 2,
786 open_duration_us: 10_000, timeout_us: 5_000,
788 sliding_window_size: 5,
789 };
790 NetworkCircuitBreaker::new(cfg).expect("valid config")
791 }
792
793 fn inject_failures(b: &mut NetworkCircuitBreaker, n: u32, ts: &mut u64) {
795 for _ in 0..n {
796 let mut g = b.call(*ts).expect("call should be permitted");
797 b.record_outcome(CircuitOutcome::Failure("err".into()), 100, *ts);
798 g.mark_recorded();
799 *ts += 1;
800 }
801 }
802
803 fn inject_successes(b: &mut NetworkCircuitBreaker, n: u32, ts: &mut u64) {
805 for _ in 0..n {
806 let mut g = b.call(*ts).expect("call should be permitted");
807 b.record_outcome(CircuitOutcome::Success, 100, *ts);
808 g.mark_recorded();
809 *ts += 1;
810 }
811 }
812
813 #[test]
816 fn test_new_starts_closed() {
817 let b = make_breaker();
818 assert!(matches!(b.state(), CircuitState::Closed { .. }));
819 }
820
821 #[test]
822 fn test_new_invalid_window_size() {
823 let cfg = CircuitConfig {
824 sliding_window_size: 0,
825 ..CircuitConfig::default()
826 };
827 assert!(matches!(
828 NetworkCircuitBreaker::new(cfg),
829 Err(BreakerError::ConfigurationError(_))
830 ));
831 }
832
833 #[test]
834 fn test_new_invalid_failure_threshold() {
835 let cfg = CircuitConfig {
836 failure_threshold: 0,
837 ..CircuitConfig::default()
838 };
839 assert!(matches!(
840 NetworkCircuitBreaker::new(cfg),
841 Err(BreakerError::ConfigurationError(_))
842 ));
843 }
844
845 #[test]
846 fn test_new_invalid_success_threshold() {
847 let cfg = CircuitConfig {
848 success_threshold: 0,
849 ..CircuitConfig::default()
850 };
851 assert!(matches!(
852 NetworkCircuitBreaker::new(cfg),
853 Err(BreakerError::ConfigurationError(_))
854 ));
855 }
856
857 #[test]
858 fn test_new_invalid_half_open_probes() {
859 let cfg = CircuitConfig {
860 half_open_probes: 0,
861 ..CircuitConfig::default()
862 };
863 assert!(matches!(
864 NetworkCircuitBreaker::new(cfg),
865 Err(BreakerError::ConfigurationError(_))
866 ));
867 }
868
869 #[test]
872 fn test_closed_allows_calls() {
873 let mut b = make_breaker();
874 let g = b.call(1000);
875 assert!(g.is_ok());
876 }
877
878 #[test]
879 fn test_closed_stays_closed_on_success() {
880 let mut b = make_breaker();
881 let mut ts = 1000u64;
882 inject_successes(&mut b, 5, &mut ts);
883 assert!(matches!(b.state(), CircuitState::Closed { .. }));
884 }
885
886 #[test]
887 fn test_closed_failure_count_increments() {
888 let mut b = make_breaker();
889 let mut ts = 1000u64;
890 inject_failures(&mut b, 2, &mut ts);
891 match b.state() {
892 CircuitState::Closed { failure_count, .. } => assert_eq!(*failure_count, 2),
893 _ => panic!("expected Closed"),
894 }
895 }
896
897 #[test]
900 fn test_closed_to_open_on_threshold() {
901 let mut b = make_breaker();
902 let mut ts = 1000u64;
903 inject_failures(&mut b, 3, &mut ts);
904 assert!(matches!(b.state(), CircuitState::Open { .. }));
905 }
906
907 #[test]
908 fn test_open_emits_threshold_event() {
909 let mut b = make_breaker();
910 let mut ts = 1000u64;
911 inject_failures(&mut b, 3, &mut ts);
912 let hist = b.event_history();
913 let has_threshold = hist
914 .iter()
915 .any(|e| matches!(e, CircuitEvent::ThresholdReached { .. }));
916 assert!(has_threshold, "expected ThresholdReached event");
917 }
918
919 #[test]
920 fn test_open_emits_state_changed_event() {
921 let mut b = make_breaker();
922 let mut ts = 1000u64;
923 inject_failures(&mut b, 3, &mut ts);
924 let hist = b.event_history();
925 let has_state_change = hist
926 .iter()
927 .any(|e| matches!(e, CircuitEvent::StateChanged { to, .. } if to == "Open"));
928 assert!(has_state_change);
929 }
930
931 #[test]
934 fn test_open_rejects_calls_before_timeout() {
935 let mut b = make_breaker();
936 let mut ts = 1000u64;
937 inject_failures(&mut b, 3, &mut ts);
938 let err = b.call(ts + 1).unwrap_err();
940 assert!(matches!(err, BreakerError::CircuitOpen { .. }));
941 }
942
943 #[test]
944 fn test_open_retry_after_reported_correctly() {
945 let mut b = make_breaker();
946 let mut ts = 1000u64;
947 inject_failures(&mut b, 3, &mut ts);
948 let call_ts = ts + 1;
949 match b.call(call_ts).unwrap_err() {
950 BreakerError::CircuitOpen { retry_after_us } => {
951 assert!(retry_after_us > 0);
954 assert!(retry_after_us <= 10_000);
955 }
956 other => panic!("unexpected error: {:?}", other),
957 }
958 }
959
960 #[test]
961 fn test_open_increments_rejection_counter() {
962 let mut b = make_breaker();
963 let mut ts = 1000u64;
964 inject_failures(&mut b, 3, &mut ts);
965 let _ = b.call(ts + 1); let m = b.metrics(ts + 1);
967 assert!(m.rejection_rate > 0.0);
968 }
969
970 #[test]
973 fn test_open_to_half_open_after_timeout() {
974 let mut b = make_breaker();
975 let mut ts = 1000u64;
976 inject_failures(&mut b, 3, &mut ts);
977 let recovery_ts = ts + 20_000;
979 let g = b.call(recovery_ts);
980 assert!(g.is_ok(), "should transition to HalfOpen");
981 assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
982 }
983
984 #[test]
985 fn test_open_to_half_open_emits_event() {
986 let mut b = make_breaker();
987 let mut ts = 1000u64;
988 inject_failures(&mut b, 3, &mut ts);
989 let _ = b.call(ts + 20_000);
990 let hist = b.event_history();
991 let has = hist
992 .iter()
993 .any(|e| matches!(e, CircuitEvent::StateChanged { to, .. } if to == "HalfOpen"));
994 assert!(has);
995 }
996
997 #[test]
1000 fn test_half_open_allows_limited_probes() {
1001 let mut b = make_breaker();
1002 let mut ts = 1000u64;
1003 inject_failures(&mut b, 3, &mut ts);
1004 let recovery_ts = ts + 20_000;
1005 let g2 = b.call(recovery_ts + 1);
1008 assert!(g2.is_ok(), "second probe should be allowed");
1009 }
1010
1011 #[test]
1012 fn test_half_open_rejects_excess_probes() {
1013 let mut b = make_breaker();
1014 let mut ts = 1000u64;
1015 inject_failures(&mut b, 3, &mut ts);
1016 let rt = ts + 20_000;
1017 let _ = b.call(rt); let _ = b.call(rt + 1); let err = b.call(rt + 2).unwrap_err();
1020 assert!(matches!(err, BreakerError::MaxProbesExceeded));
1021 }
1022
1023 #[test]
1026 fn test_half_open_to_closed_on_success() {
1027 let mut b = make_breaker();
1028 let mut ts = 1000u64;
1029 inject_failures(&mut b, 3, &mut ts);
1030 let rt = ts + 20_000;
1031 let mut g = b.call(rt).expect("first probe");
1033 b.record_outcome(CircuitOutcome::Success, 50, rt);
1034 g.mark_recorded();
1035 let mut g2 = b.call(rt + 1).expect("second probe");
1037 let evt = b.record_outcome(CircuitOutcome::Success, 50, rt + 1);
1038 g2.mark_recorded();
1039 assert!(
1040 matches!(evt, Some(CircuitEvent::RecoverySucceeded)),
1041 "expected RecoverySucceeded"
1042 );
1043 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1044 }
1045
1046 #[test]
1047 fn test_recovery_succeeded_event_in_history() {
1048 let mut b = make_breaker();
1049 let mut ts = 1000u64;
1050 inject_failures(&mut b, 3, &mut ts);
1051 let rt = ts + 20_000;
1052 let mut g = b.call(rt).expect("probe 1");
1053 b.record_outcome(CircuitOutcome::Success, 50, rt);
1054 g.mark_recorded();
1055 let mut g2 = b.call(rt + 1).expect("probe 2");
1056 b.record_outcome(CircuitOutcome::Success, 50, rt + 1);
1057 g2.mark_recorded();
1058 let hist = b.event_history();
1059 let has = hist
1060 .iter()
1061 .any(|e| matches!(e, CircuitEvent::RecoverySucceeded));
1062 assert!(has);
1063 }
1064
1065 #[test]
1068 fn test_half_open_to_open_on_failure() {
1069 let mut b = make_breaker();
1070 let mut ts = 1000u64;
1071 inject_failures(&mut b, 3, &mut ts);
1072 let rt = ts + 20_000;
1073 let mut g = b.call(rt).expect("probe");
1074 let evt = b.record_outcome(CircuitOutcome::Failure("boom".into()), 200, rt);
1075 g.mark_recorded();
1076 assert!(matches!(evt, Some(CircuitEvent::RecoveryFailed)));
1077 assert!(matches!(b.state(), CircuitState::Open { .. }));
1078 }
1079
1080 #[test]
1081 fn test_half_open_timeout_reopens() {
1082 let mut b = make_breaker();
1083 let mut ts = 1000u64;
1084 inject_failures(&mut b, 3, &mut ts);
1085 let rt = ts + 20_000;
1086 let mut g = b.call(rt).expect("probe");
1087 let evt = b.record_outcome(CircuitOutcome::Timeout, 9_999, rt);
1088 g.mark_recorded();
1089 assert!(matches!(evt, Some(CircuitEvent::RecoveryFailed)));
1090 assert!(matches!(b.state(), CircuitState::Open { .. }));
1091 }
1092
1093 #[test]
1094 fn test_recovery_failed_event_in_history() {
1095 let mut b = make_breaker();
1096 let mut ts = 1000u64;
1097 inject_failures(&mut b, 3, &mut ts);
1098 let rt = ts + 20_000;
1099 let mut g = b.call(rt).expect("probe");
1100 b.record_outcome(CircuitOutcome::Failure("x".into()), 10, rt);
1101 g.mark_recorded();
1102 let hist = b.event_history();
1103 assert!(hist
1104 .iter()
1105 .any(|e| matches!(e, CircuitEvent::RecoveryFailed)));
1106 }
1107
1108 #[test]
1111 fn test_full_state_cycle_closed_open_halfopen_closed() {
1112 let mut b = make_breaker();
1113 let mut ts = 0u64;
1114
1115 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1117
1118 inject_failures(&mut b, 3, &mut ts);
1120 assert!(matches!(b.state(), CircuitState::Open { .. }));
1121
1122 ts += 20_000;
1124
1125 let mut g = b.call(ts).expect("probe 1");
1127 b.record_outcome(CircuitOutcome::Success, 10, ts);
1128 g.mark_recorded();
1129 assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
1130
1131 ts += 1;
1133 let mut g2 = b.call(ts).expect("probe 2");
1134 b.record_outcome(CircuitOutcome::Success, 10, ts);
1135 g2.mark_recorded();
1136 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1137 }
1138
1139 #[test]
1140 fn test_multiple_trip_recover_cycles() {
1141 let mut b = make_breaker();
1142 let mut ts = 0u64;
1143
1144 for cycle in 0..3u32 {
1145 inject_failures(&mut b, 3, &mut ts);
1147 assert!(
1148 matches!(b.state(), CircuitState::Open { .. }),
1149 "cycle {}",
1150 cycle
1151 );
1152
1153 ts += 20_000;
1155 let mut g = b.call(ts).expect("probe 1");
1156 b.record_outcome(CircuitOutcome::Success, 10, ts);
1157 g.mark_recorded();
1158 ts += 1;
1159 let mut g2 = b.call(ts).expect("probe 2");
1160 b.record_outcome(CircuitOutcome::Success, 10, ts);
1161 g2.mark_recorded();
1162 assert!(
1163 matches!(b.state(), CircuitState::Closed { .. }),
1164 "cycle {} after recovery",
1165 cycle
1166 );
1167 ts += 1;
1168 }
1169 }
1170
1171 #[test]
1174 fn test_sliding_window_evicts_oldest() {
1175 let mut b = make_breaker();
1178 let mut ts = 0u64;
1179 inject_successes(&mut b, 5, &mut ts);
1180 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1181 inject_failures(&mut b, 3, &mut ts);
1182 assert!(matches!(b.state(), CircuitState::Open { .. }));
1183 }
1184
1185 #[test]
1186 fn test_sliding_window_does_not_trip_if_failures_old() {
1187 let mut b = make_breaker();
1191 let mut ts = 0u64;
1192 inject_failures(&mut b, 2, &mut ts);
1193 inject_successes(&mut b, 5, &mut ts);
1195 inject_failures(&mut b, 2, &mut ts);
1197 assert!(
1198 matches!(b.state(), CircuitState::Closed { .. }),
1199 "window should have evicted old failures"
1200 );
1201 }
1202
1203 #[test]
1204 fn test_sliding_window_failure_rate() {
1205 let mut sw = SlidingWindow::new(4);
1206 sw.push(false);
1207 sw.push(false);
1208 sw.push(true);
1209 sw.push(true);
1210 assert!((sw.failure_rate() - 0.5).abs() < f64::EPSILON);
1211 assert!((sw.success_rate() - 0.5).abs() < f64::EPSILON);
1212 }
1213
1214 #[test]
1215 fn test_sliding_window_evicts_when_full() {
1216 let mut sw = SlidingWindow::new(3);
1217 sw.push(false); sw.push(true);
1219 sw.push(true);
1220 sw.push(true); assert_eq!(sw.failure_count, 0);
1222 assert_eq!(sw.success_count, 3);
1223 }
1224
1225 #[test]
1226 fn test_sliding_window_empty_failure_rate() {
1227 let sw = SlidingWindow::new(5);
1228 assert!((sw.failure_rate() - 0.0).abs() < f64::EPSILON);
1229 assert!((sw.success_rate() - 1.0).abs() < f64::EPSILON);
1230 }
1231
1232 #[test]
1235 fn test_force_open_from_closed() {
1236 let mut b = make_breaker();
1237 b.force_open(5000);
1238 assert!(matches!(b.state(), CircuitState::Open { .. }));
1239 }
1240
1241 #[test]
1242 fn test_force_close_from_open() {
1243 let mut b = make_breaker();
1244 let mut ts = 0u64;
1245 inject_failures(&mut b, 3, &mut ts);
1246 b.force_close();
1247 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1248 }
1249
1250 #[test]
1251 fn test_force_open_from_half_open() {
1252 let mut b = make_breaker();
1253 let mut ts = 0u64;
1254 inject_failures(&mut b, 3, &mut ts);
1255 let _ = b.call(ts + 20_000);
1256 b.force_open(ts + 25_000);
1257 assert!(matches!(b.state(), CircuitState::Open { .. }));
1258 }
1259
1260 #[test]
1261 fn test_force_open_emits_event() {
1262 let mut b = make_breaker();
1263 b.force_open(999);
1264 let hist = b.event_history();
1265 assert!(hist.iter().any(|e| {
1266 matches!(e, CircuitEvent::StateChanged { to, at, .. } if to == "Open" && *at == 999)
1267 }));
1268 }
1269
1270 #[test]
1271 fn test_force_close_emits_event() {
1272 let mut b = make_breaker();
1273 let mut ts = 0u64;
1274 inject_failures(&mut b, 3, &mut ts);
1275 b.force_close();
1276 let hist = b.event_history();
1277 assert!(hist
1278 .iter()
1279 .any(|e| { matches!(e, CircuitEvent::StateChanged { to, .. } if to == "Closed") }));
1280 }
1281
1282 #[test]
1283 fn test_force_close_resets_window() {
1284 let mut b = make_breaker();
1285 let mut ts = 0u64;
1286 inject_failures(&mut b, 2, &mut ts);
1287 b.force_close();
1288 inject_failures(&mut b, 2, &mut ts);
1291 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1292 }
1293
1294 #[test]
1297 fn test_metrics_initial_state() {
1298 let b = make_breaker();
1299 let m = b.metrics(0);
1300 assert_eq!(m.total_requests, 0);
1301 assert_eq!(m.current_state, "Closed");
1302 assert!((m.failure_rate - 0.0).abs() < f64::EPSILON);
1303 }
1304
1305 #[test]
1306 fn test_metrics_success_rate() {
1307 let mut b = make_breaker();
1308 let mut ts = 0u64;
1309 inject_successes(&mut b, 4, &mut ts);
1310 let m = b.metrics(ts);
1311 assert!((m.success_rate - 1.0).abs() < f64::EPSILON);
1312 assert!((m.failure_rate - 0.0).abs() < f64::EPSILON);
1313 }
1314
1315 #[test]
1316 fn test_metrics_failure_rate() {
1317 let mut b = make_breaker();
1318 let mut ts = 0u64;
1319 inject_failures(&mut b, 2, &mut ts);
1320 inject_successes(&mut b, 2, &mut ts);
1321 let m = b.metrics(ts);
1322 assert!((m.failure_rate - 0.5).abs() < f64::EPSILON);
1323 }
1324
1325 #[test]
1326 fn test_metrics_rejection_rate() {
1327 let mut b = make_breaker();
1328 let mut ts = 0u64;
1329 inject_failures(&mut b, 3, &mut ts);
1330 let _ = b.call(ts + 1);
1332 let m = b.metrics(ts + 1);
1335 assert!(m.rejection_rate > 0.0);
1336 }
1337
1338 #[test]
1339 fn test_metrics_avg_response_time() {
1340 let mut b = make_breaker();
1341 let mut ts = 0u64;
1342 let mut g1 = b.call(ts).expect("call");
1343 b.record_outcome(CircuitOutcome::Success, 100, ts);
1344 g1.mark_recorded();
1345 ts += 1;
1346 let mut g2 = b.call(ts).expect("call");
1347 b.record_outcome(CircuitOutcome::Success, 300, ts);
1348 g2.mark_recorded();
1349 let m = b.metrics(ts);
1350 assert!((m.avg_response_time_us - 200.0).abs() < f64::EPSILON);
1351 }
1352
1353 #[test]
1354 fn test_reset_metrics() {
1355 let mut b = make_breaker();
1356 let mut ts = 0u64;
1357 inject_successes(&mut b, 3, &mut ts);
1358 b.reset_metrics();
1359 let m = b.metrics(ts);
1360 assert_eq!(m.total_requests, 0);
1361 }
1362
1363 #[test]
1364 fn test_metrics_current_state_open() {
1365 let mut b = make_breaker();
1366 let mut ts = 0u64;
1367 inject_failures(&mut b, 3, &mut ts);
1368 let m = b.metrics(ts);
1369 assert_eq!(m.current_state, "Open");
1370 }
1371
1372 #[test]
1373 fn test_metrics_current_state_half_open() {
1374 let mut b = make_breaker();
1375 let mut ts = 0u64;
1376 inject_failures(&mut b, 3, &mut ts);
1377 let rt = ts + 20_000;
1378 let mut g = b.call(rt).expect("probe");
1379 b.record_outcome(CircuitOutcome::Success, 10, rt);
1380 g.mark_recorded();
1381 let m = b.metrics(rt);
1382 assert_eq!(m.current_state, "HalfOpen");
1383 }
1384
1385 #[test]
1388 fn test_event_history_empty_initially() {
1389 let b = make_breaker();
1390 assert!(b.event_history().is_empty());
1391 }
1392
1393 #[test]
1394 fn test_event_history_capped_at_50() {
1395 let cfg = CircuitConfig {
1397 failure_threshold: 1,
1398 success_threshold: 1,
1399 half_open_probes: 1,
1400 open_duration_us: 1,
1401 timeout_us: 1_000_000,
1402 sliding_window_size: 1,
1403 };
1404 let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1405 let mut ts = 0u64;
1406
1407 for _ in 0..60u32 {
1408 b.force_close();
1410 let mut g = b.call(ts).expect("call");
1412 b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1413 g.mark_recorded();
1414 ts += 2;
1415 let mut g2 = b.call(ts).expect("probe");
1417 b.record_outcome(CircuitOutcome::Success, 10, ts);
1418 g2.mark_recorded();
1419 ts += 2;
1420 }
1421
1422 let hist = b.event_history();
1423 assert!(
1424 hist.len() <= 50,
1425 "history must not exceed 50 events, got {}",
1426 hist.len()
1427 );
1428 }
1429
1430 #[test]
1431 fn test_event_history_records_all_transitions() {
1432 let mut b = make_breaker();
1433 let mut ts = 0u64;
1434 inject_failures(&mut b, 3, &mut ts);
1435 ts += 20_000;
1436 let mut g = b.call(ts).expect("probe 1");
1437 b.record_outcome(CircuitOutcome::Success, 10, ts);
1438 g.mark_recorded();
1439 ts += 1;
1440 let mut g2 = b.call(ts).expect("probe 2");
1441 b.record_outcome(CircuitOutcome::Success, 10, ts);
1442 g2.mark_recorded();
1443
1444 let hist = b.event_history();
1445 let labels: Vec<&str> = hist
1446 .iter()
1447 .filter_map(|e| {
1448 if let CircuitEvent::StateChanged { to, .. } = e {
1449 Some(to.as_str())
1450 } else {
1451 None
1452 }
1453 })
1454 .collect();
1455
1456 assert!(labels.contains(&"Open"), "should have Open transition");
1457 assert!(
1458 labels.contains(&"HalfOpen"),
1459 "should have HalfOpen transition"
1460 );
1461 assert!(labels.contains(&"Closed"), "should have Closed transition");
1462 }
1463
1464 #[test]
1467 fn test_guard_mark_recorded() {
1468 let mut g = CircuitCallGuard::new(0);
1469 assert!(!g.is_recorded());
1470 g.mark_recorded();
1471 assert!(g.is_recorded());
1472 }
1473
1474 #[test]
1475 fn test_guard_issued_at() {
1476 let g = CircuitCallGuard::new(42_000);
1477 assert_eq!(g.issued_at, 42_000);
1478 }
1479
1480 #[test]
1481 fn test_guard_drop_without_recording() {
1482 {
1484 let _g = CircuitCallGuard::new(100);
1485 }
1487 }
1488
1489 #[test]
1492 fn test_xorshift64_deterministic() {
1493 let mut state = 12345u64;
1494 let v1 = xorshift64(&mut state);
1495 let mut state2 = 12345u64;
1496 let v2 = xorshift64(&mut state2);
1497 assert_eq!(v1, v2);
1498 }
1499
1500 #[test]
1501 fn test_xorshift64_non_zero() {
1502 let mut state = 1u64;
1503 for _ in 0..100 {
1504 let v = xorshift64(&mut state);
1505 assert_ne!(v, 0);
1506 }
1507 }
1508
1509 #[test]
1510 fn test_xorshift64_distinct_values() {
1511 let mut state = 999u64;
1512 let a = xorshift64(&mut state);
1513 let b = xorshift64(&mut state);
1514 assert_ne!(a, b);
1515 }
1516
1517 #[test]
1520 fn test_breaker_error_display_circuit_open() {
1521 let e = BreakerError::CircuitOpen {
1522 retry_after_us: 5000,
1523 };
1524 let s = format!("{}", e);
1525 assert!(s.contains("5000"));
1526 }
1527
1528 #[test]
1529 fn test_breaker_error_display_max_probes() {
1530 let e = BreakerError::MaxProbesExceeded;
1531 let s = format!("{}", e);
1532 assert!(!s.is_empty());
1533 }
1534
1535 #[test]
1536 fn test_breaker_error_display_config() {
1537 let e = BreakerError::ConfigurationError("bad value".into());
1538 let s = format!("{}", e);
1539 assert!(s.contains("bad value"));
1540 }
1541
1542 #[test]
1545 fn test_record_outcome_in_open_state_no_panic() {
1546 let mut b = make_breaker();
1547 let mut ts = 0u64;
1548 inject_failures(&mut b, 3, &mut ts);
1549 let evt = b.record_outcome(CircuitOutcome::Success, 10, ts);
1551 assert!(evt.is_none(), "no state change expected");
1552 assert!(matches!(b.state(), CircuitState::Open { .. }));
1553 }
1554
1555 #[test]
1556 fn test_exact_boundary_open_to_half_open() {
1557 let mut b = make_breaker();
1558 let mut ts = 1000u64;
1559 inject_failures(&mut b, 3, &mut ts);
1560 if let CircuitState::Open {
1564 opened_at,
1565 retry_after_us,
1566 } = b.state().clone()
1567 {
1568 let exact_boundary = opened_at + retry_after_us;
1569 let err = b.call(exact_boundary - 1).unwrap_err();
1571 assert!(matches!(err, BreakerError::CircuitOpen { .. }));
1572 let g = b.call(exact_boundary);
1574 assert!(g.is_ok());
1575 } else {
1576 panic!("expected Open state");
1577 }
1578 }
1579
1580 #[test]
1581 fn test_half_open_probe_count_increments_on_call() {
1582 let mut b = make_breaker();
1583 let mut ts = 0u64;
1584 inject_failures(&mut b, 3, &mut ts);
1585 let rt = ts + 20_000;
1586 let mut g = b.call(rt).expect("probe 1");
1587 match b.state() {
1589 CircuitState::HalfOpen { probe_count, .. } => assert_eq!(*probe_count, 1),
1590 _ => panic!("expected HalfOpen"),
1591 }
1592 g.mark_recorded();
1593 let _g2 = b.call(rt + 1).expect("probe 2");
1595 match b.state() {
1596 CircuitState::HalfOpen { probe_count, .. } => assert_eq!(*probe_count, 2),
1597 _ => panic!("expected HalfOpen"),
1598 }
1599 }
1600
1601 #[test]
1602 fn test_no_false_trip_on_mixed_outcomes() {
1603 let cfg = CircuitConfig {
1607 failure_threshold: 6, success_threshold: 2,
1609 half_open_probes: 2,
1610 open_duration_us: 10_000,
1611 timeout_us: 5_000,
1612 sliding_window_size: 10,
1613 };
1614 let mut b = NetworkCircuitBreaker::new(cfg).expect("valid config");
1615 let mut ts = 0u64;
1616 for _ in 0..10u32 {
1620 let mut g = b.call(ts).expect("call");
1621 b.record_outcome(CircuitOutcome::Success, 50, ts);
1622 g.mark_recorded();
1623 ts += 1;
1624 let mut g2 = b.call(ts).expect("call");
1625 b.record_outcome(CircuitOutcome::Failure("transient".into()), 100, ts);
1626 g2.mark_recorded();
1627 ts += 1;
1628 }
1629 assert!(
1630 matches!(b.state(), CircuitState::Closed { .. }),
1631 "circuit should remain Closed with 50% failure rate below threshold"
1632 );
1633 }
1634
1635 #[test]
1636 fn test_rejected_outcome_does_not_affect_window() {
1637 let mut b = make_breaker();
1638 let mut ts = 0u64;
1639 inject_failures(&mut b, 3, &mut ts);
1640 let _ = b.call(ts + 1);
1642 let _ = b.call(ts + 2);
1643 b.force_close();
1645 inject_failures(&mut b, 2, &mut ts);
1647 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1648 }
1649
1650 #[test]
1651 fn test_ncb_aliases_are_correct_types() {
1652 let cfg: NcbCircuitConfig = CircuitConfig::default();
1654 let b = NetworkCircuitBreaker::new(cfg).expect("valid");
1655 let state_ref: &NcbCircuitState = b.state();
1656 assert!(matches!(state_ref, CircuitState::Closed { .. }));
1657 }
1658
1659 #[test]
1660 fn test_large_window_partial_fill() {
1661 let cfg = CircuitConfig {
1662 failure_threshold: 3,
1663 success_threshold: 2,
1664 half_open_probes: 2,
1665 open_duration_us: 10_000,
1666 timeout_us: 5_000,
1667 sliding_window_size: 100,
1668 };
1669 let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1670 let mut ts = 0u64;
1671 inject_failures(&mut b, 2, &mut ts);
1673 assert!(matches!(b.state(), CircuitState::Closed { .. }));
1674 let mut g = b.call(ts).expect("call before trip");
1676 b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1677 g.mark_recorded();
1678 let _ts_next = ts + 1; assert!(
1680 matches!(b.state(), CircuitState::Open { .. }),
1681 "circuit should be Open after 3rd failure"
1682 );
1683 }
1684
1685 #[test]
1686 fn test_metrics_accumulator_timeout_counted_as_failure() {
1687 let mut acc = MetricsAccumulator::default();
1688 acc.record_outcome(&CircuitOutcome::Timeout, 999);
1689 assert_eq!(acc.total_timeouts, 1);
1690 assert_eq!(acc.total_failures, 1);
1691 assert_eq!(acc.total_requests, 1);
1692 }
1693
1694 #[test]
1695 fn test_metrics_accumulator_rejected_not_in_response_time() {
1696 let mut acc = MetricsAccumulator::default();
1697 acc.record_outcome(&CircuitOutcome::Rejected, 0);
1698 assert_eq!(acc.response_time_samples, 0);
1699 assert_eq!(acc.total_rejections, 1);
1700 }
1701
1702 #[test]
1703 fn test_open_duration_zero_immediate_halfopen() {
1704 let cfg = CircuitConfig {
1705 failure_threshold: 1,
1706 success_threshold: 1,
1707 half_open_probes: 1,
1708 open_duration_us: 0,
1709 timeout_us: 1_000_000,
1710 sliding_window_size: 1,
1711 };
1712 let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1713 let ts = 0u64;
1714 let mut g = b.call(ts).expect("call");
1715 b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1716 g.mark_recorded();
1717 assert!(matches!(b.state(), CircuitState::Open { .. }));
1718 let g2 = b.call(ts);
1720 assert!(g2.is_ok());
1721 assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
1722 }
1723}