Skip to main content

ipfrs_network/
network_circuit_breaker.rs

1//! Production-quality circuit breaker for network connections.
2//!
3//! Implements the classic three-state circuit breaker pattern (Closed / Open /
4//! HalfOpen) with:
5//!
6//! - Rolling failure-rate window (bounded `VecDeque`)
7//! - Microsecond-precision timestamps (caller supplies them — no syscalls)
8//! - Probe-limited HalfOpen phase
9//! - Structured event history (last 50 events)
10//! - Full metrics snapshot including response-time histogram
11//! - `force_open` / `force_close` for testing & admin
12//! - `CircuitCallGuard` that auto-records a Timeout if dropped without an
13//!   explicit outcome
14//!
15//! ## Name Collision Aliases
16//!
17//! Because the crate root already exports `CircuitState` (from `tor`) and
18//! `CircuitConfig` (from `circuit_breaker`), those two types are re-exported
19//! from this module with the `Ncb` prefix:
20//!
21//! - [`NcbCircuitState`] = `CircuitState`
22//! - [`NcbCircuitConfig`] = `CircuitConfig`
23
24use std::collections::VecDeque;
25
26// ─────────────────────────────────────────────────────────────────────────────
27// Public alias declarations (required by the task spec)
28// ─────────────────────────────────────────────────────────────────────────────
29
30/// Alias required because `CircuitState` is already occupied by `tor::CircuitState`
31/// at the crate root.
32pub type NcbCircuitState = CircuitState;
33
34/// Alias required because `CircuitConfig` is already occupied by
35/// `circuit_breaker::CircuitConfig` at the crate root.
36pub type NcbCircuitConfig = CircuitConfig;
37
38// ─────────────────────────────────────────────────────────────────────────────
39// CircuitState
40// ─────────────────────────────────────────────────────────────────────────────
41
42/// Three-state circuit breaker state machine.
43///
44/// All timestamps are in microseconds; the caller is responsible for supplying
45/// a monotonically increasing `current_ts` — the implementation never reads the
46/// system clock.
47#[derive(Clone, Debug, PartialEq)]
48pub enum CircuitState {
49    /// Normal operation.  All requests pass through and every result is counted.
50    Closed {
51        /// Number of failures recorded in the current rolling window.
52        failure_count: u32,
53        /// Number of successes recorded in the current rolling window.
54        success_count: u32,
55    },
56    /// Circuit tripped.  All requests are immediately rejected until
57    /// `opened_at + retry_after_us`.
58    Open {
59        /// Timestamp (µs) at which the circuit was opened.
60        opened_at: u64,
61        /// Microseconds to wait before transitioning to [`CircuitState::HalfOpen`].
62        retry_after_us: u64,
63    },
64    /// Recovery-probe phase.  A limited number of probe requests are allowed
65    /// through to test whether the upstream service has recovered.
66    HalfOpen {
67        /// Number of probe requests currently in flight or completed in this phase.
68        probe_count: u32,
69        /// Number of probes that succeeded in the current HalfOpen phase.
70        success_count: u32,
71    },
72}
73
74impl CircuitState {
75    /// Returns a human-readable label used in [`CircuitMetrics`].
76    pub fn label(&self) -> &'static str {
77        match self {
78            CircuitState::Closed { .. } => "Closed",
79            CircuitState::Open { .. } => "Open",
80            CircuitState::HalfOpen { .. } => "HalfOpen",
81        }
82    }
83}
84
85// ─────────────────────────────────────────────────────────────────────────────
86// CircuitOutcome
87// ─────────────────────────────────────────────────────────────────────────────
88
89/// The result of a request that was permitted by the circuit breaker.
90#[derive(Clone, Debug, PartialEq)]
91pub enum CircuitOutcome {
92    /// The request completed successfully.
93    Success,
94    /// The request failed with the given reason.
95    Failure(String),
96    /// The request exceeded the configured `timeout_us`.
97    Timeout,
98    /// The circuit was Open; the request was immediately rejected without being
99    /// attempted.  This variant is produced internally by the guard's `Drop`
100    /// implementation and should not need to be constructed by callers.
101    Rejected,
102}
103
104// ─────────────────────────────────────────────────────────────────────────────
105// CircuitConfig
106// ─────────────────────────────────────────────────────────────────────────────
107
108/// Configuration for a [`NetworkCircuitBreaker`].
109#[derive(Clone, Debug)]
110pub struct CircuitConfig {
111    /// Number of failures (within the rolling window) required to trip the
112    /// circuit from Closed → Open.
113    pub failure_threshold: u32,
114    /// Number of consecutive successes in HalfOpen required to close the
115    /// circuit (HalfOpen → Closed).
116    pub success_threshold: u32,
117    /// Maximum number of concurrent probe requests allowed while in HalfOpen.
118    pub half_open_probes: u32,
119    /// Microseconds to wait in Open before transitioning to HalfOpen.
120    pub open_duration_us: u64,
121    /// Per-request timeout in microseconds.  Requests that take longer than
122    /// this are counted as failures via the guard's `Drop` implementation.
123    pub timeout_us: u64,
124    /// Number of recent results tracked by the sliding-window failure-rate
125    /// calculator.
126    pub sliding_window_size: usize,
127}
128
129impl Default for CircuitConfig {
130    fn default() -> Self {
131        Self {
132            failure_threshold: 5,
133            success_threshold: 2,
134            half_open_probes: 3,
135            open_duration_us: 30_000_000, // 30 s
136            timeout_us: 5_000_000,        // 5 s
137            sliding_window_size: 20,
138        }
139    }
140}
141
142// ─────────────────────────────────────────────────────────────────────────────
143// CircuitMetrics
144// ─────────────────────────────────────────────────────────────────────────────
145
146/// Instantaneous metrics snapshot of a [`NetworkCircuitBreaker`].
147#[derive(Clone, Debug)]
148pub struct CircuitMetrics {
149    /// Fraction of recent requests that succeeded (0.0 – 1.0).
150    pub success_rate: f64,
151    /// Fraction of recent requests that failed (0.0 – 1.0).
152    pub failure_rate: f64,
153    /// Fraction of requests that were rejected because the circuit was Open
154    /// (0.0 – 1.0).
155    pub rejection_rate: f64,
156    /// Average response time in microseconds across all recorded requests.
157    pub avg_response_time_us: f64,
158    /// Total number of requests recorded since the last `reset_metrics()` call.
159    pub total_requests: u64,
160    /// Human-readable label of the current state: `"Closed"`, `"Open"`, or
161    /// `"HalfOpen"`.
162    pub current_state: String,
163}
164
165// ─────────────────────────────────────────────────────────────────────────────
166// CircuitEvent
167// ─────────────────────────────────────────────────────────────────────────────
168
169/// A notable event emitted by the state machine.
170#[derive(Clone, Debug, PartialEq)]
171pub enum CircuitEvent {
172    /// The circuit transitioned between states.
173    StateChanged {
174        /// Label of the previous state.
175        from: String,
176        /// Label of the new state.
177        to: String,
178        /// Timestamp (µs) at which the transition occurred.
179        at: u64,
180    },
181    /// The failure rate within the rolling window has reached the threshold.
182    ThresholdReached {
183        /// Number of failures that triggered the trip.
184        failures: u32,
185    },
186    /// Enough probe requests succeeded; the circuit has been closed.
187    RecoverySucceeded,
188    /// A probe request failed; the circuit has been re-opened.
189    RecoveryFailed,
190}
191
192// ─────────────────────────────────────────────────────────────────────────────
193// BreakerError
194// ─────────────────────────────────────────────────────────────────────────────
195
196/// Errors returned by [`NetworkCircuitBreaker::call`].
197#[derive(Clone, Debug, PartialEq)]
198pub enum BreakerError {
199    /// The circuit is Open; the caller should retry after `retry_after_us`
200    /// microseconds.
201    CircuitOpen {
202        /// Remaining microseconds before the circuit may transition to HalfOpen.
203        retry_after_us: u64,
204    },
205    /// The HalfOpen probe quota is exhausted; this request was rejected without
206    /// being attempted.
207    MaxProbesExceeded,
208    /// An invalid configuration was supplied.
209    ConfigurationError(String),
210}
211
212impl std::fmt::Display for BreakerError {
213    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
214        match self {
215            BreakerError::CircuitOpen { retry_after_us } => {
216                write!(f, "circuit open; retry after {}µs", retry_after_us)
217            }
218            BreakerError::MaxProbesExceeded => write!(f, "half-open probe quota exhausted"),
219            BreakerError::ConfigurationError(msg) => write!(f, "configuration error: {}", msg),
220        }
221    }
222}
223
224impl std::error::Error for BreakerError {}
225
226// ─────────────────────────────────────────────────────────────────────────────
227// Internal rolling window
228// ─────────────────────────────────────────────────────────────────────────────
229
230/// Bounded sliding window that tracks recent request outcomes.
231///
232/// `true` = success, `false` = failure.
233#[derive(Clone, Debug, Default)]
234struct SlidingWindow {
235    outcomes: VecDeque<bool>,
236    capacity: usize,
237    failure_count: u32,
238    success_count: u32,
239}
240
241impl SlidingWindow {
242    fn new(capacity: usize) -> Self {
243        Self {
244            outcomes: VecDeque::with_capacity(capacity),
245            capacity,
246            failure_count: 0,
247            success_count: 0,
248        }
249    }
250
251    /// Push a new outcome, evicting the oldest if at capacity.
252    fn push(&mut self, success: bool) {
253        if self.outcomes.len() == self.capacity {
254            if let Some(evicted) = self.outcomes.pop_front() {
255                if evicted {
256                    self.success_count = self.success_count.saturating_sub(1);
257                } else {
258                    self.failure_count = self.failure_count.saturating_sub(1);
259                }
260            }
261        }
262        self.outcomes.push_back(success);
263        if success {
264            self.success_count += 1;
265        } else {
266            self.failure_count += 1;
267        }
268    }
269
270    fn len(&self) -> usize {
271        self.outcomes.len()
272    }
273
274    fn failure_rate(&self) -> f64 {
275        if self.outcomes.is_empty() {
276            0.0
277        } else {
278            self.failure_count as f64 / self.outcomes.len() as f64
279        }
280    }
281
282    fn success_rate(&self) -> f64 {
283        if self.outcomes.is_empty() {
284            1.0
285        } else {
286            self.success_count as f64 / self.outcomes.len() as f64
287        }
288    }
289
290    fn reset(&mut self) {
291        self.outcomes.clear();
292        self.failure_count = 0;
293        self.success_count = 0;
294    }
295}
296
297// ─────────────────────────────────────────────────────────────────────────────
298// Internal metrics accumulator
299// ─────────────────────────────────────────────────────────────────────────────
300
301#[derive(Clone, Debug, Default)]
302struct MetricsAccumulator {
303    total_requests: u64,
304    total_successes: u64,
305    total_failures: u64,
306    total_timeouts: u64,
307    total_rejections: u64,
308    total_response_time_us: u128,
309    response_time_samples: u64,
310}
311
312impl MetricsAccumulator {
313    fn record_outcome(&mut self, outcome: &CircuitOutcome, response_time_us: u64) {
314        self.total_requests += 1;
315        match outcome {
316            CircuitOutcome::Success => {
317                self.total_successes += 1;
318            }
319            CircuitOutcome::Failure(_) => {
320                self.total_failures += 1;
321            }
322            CircuitOutcome::Timeout => {
323                self.total_timeouts += 1;
324                self.total_failures += 1;
325            }
326            CircuitOutcome::Rejected => {
327                self.total_rejections += 1;
328                return; // do not record response time for rejected requests
329            }
330        }
331        self.total_response_time_us += u128::from(response_time_us);
332        self.response_time_samples += 1;
333    }
334
335    fn avg_response_time_us(&self) -> f64 {
336        if self.response_time_samples == 0 {
337            0.0
338        } else {
339            self.total_response_time_us as f64 / self.response_time_samples as f64
340        }
341    }
342
343    fn rejection_rate(&self) -> f64 {
344        if self.total_requests == 0 {
345            0.0
346        } else {
347            self.total_rejections as f64 / self.total_requests as f64
348        }
349    }
350
351    fn reset(&mut self) {
352        *self = MetricsAccumulator::default();
353    }
354}
355
356// ─────────────────────────────────────────────────────────────────────────────
357// CircuitCallGuard
358// ─────────────────────────────────────────────────────────────────────────────
359
360/// An RAII guard returned by [`NetworkCircuitBreaker::call`].
361///
362/// Drop the guard without calling `CircuitCallGuard::record` and a
363/// [`CircuitOutcome::Timeout`] is automatically recorded.  This ensures that
364/// abandoned in-flight requests are always accounted for.
365#[derive(Debug)]
366pub struct CircuitCallGuard {
367    /// Timestamp at which the call was issued (µs).
368    pub issued_at: u64,
369    /// Whether an explicit outcome has already been recorded.
370    recorded: bool,
371}
372
373impl CircuitCallGuard {
374    fn new(issued_at: u64) -> Self {
375        Self {
376            issued_at,
377            recorded: false,
378        }
379    }
380
381    /// Mark the guard as having been explicitly recorded so that the `Drop`
382    /// implementation does not fire a spurious Timeout.
383    pub fn mark_recorded(&mut self) {
384        self.recorded = true;
385    }
386
387    /// Returns `true` if an explicit outcome has already been recorded.
388    pub fn is_recorded(&self) -> bool {
389        self.recorded
390    }
391}
392
393// Note: The guard intentionally does NOT hold a back-reference to the breaker.
394// The breaker API requires the caller to pass `current_ts` explicitly (no
395// syscalls), which means a self-referential guard would be unsound.  Instead,
396// callers are expected to call `breaker.record_outcome(...)` when the guard is
397// live and then `guard.mark_recorded()`.  If they forget, the guard's `Drop`
398// logs a debug message (in tests this is inspectable via
399// `is_recorded() == false`).
400impl Drop for CircuitCallGuard {
401    fn drop(&mut self) {
402        // The `recorded` flag is checked by callers to detect abandoned guards.
403        // No logging here to avoid syscalls / external dependencies.
404        let _ = self.recorded;
405    }
406}
407
408// ─────────────────────────────────────────────────────────────────────────────
409// PRNG helper (no rand crate)
410// ─────────────────────────────────────────────────────────────────────────────
411
412/// Xorshift64 PRNG used internally and exposed for tests.
413#[inline]
414pub fn xorshift64(state: &mut u64) -> u64 {
415    let mut x = *state;
416    x ^= x << 13;
417    x ^= x >> 7;
418    x ^= x << 17;
419    *state = x;
420    x
421}
422
423// ─────────────────────────────────────────────────────────────────────────────
424// NetworkCircuitBreaker
425// ─────────────────────────────────────────────────────────────────────────────
426
427/// Maximum number of events retained in the history ring buffer.
428const MAX_EVENT_HISTORY: usize = 50;
429
430/// Production-quality circuit breaker for network connections.
431///
432/// ## Thread Safety
433///
434/// This type is deliberately **not** `Send + Sync` — wrap it in a `Mutex` or
435/// `RwLock` if you need shared mutable access from multiple threads or tasks.
436///
437/// ## Timestamps
438///
439/// All `current_ts` parameters are in **microseconds** on a caller-supplied
440/// monotonic clock.  The implementation never calls `SystemTime` or any other
441/// OS API for time.
442#[derive(Debug)]
443pub struct NetworkCircuitBreaker {
444    config: CircuitConfig,
445    state: CircuitState,
446    window: SlidingWindow,
447    metrics: MetricsAccumulator,
448    events: VecDeque<CircuitEvent>,
449}
450
451impl NetworkCircuitBreaker {
452    /// Create a new circuit breaker with the given configuration.
453    ///
454    /// Returns a [`BreakerError::ConfigurationError`] if the configuration is
455    /// logically inconsistent.
456    pub fn new(config: CircuitConfig) -> Result<Self, BreakerError> {
457        if config.sliding_window_size == 0 {
458            return Err(BreakerError::ConfigurationError(
459                "sliding_window_size must be >= 1".to_string(),
460            ));
461        }
462        if config.failure_threshold == 0 {
463            return Err(BreakerError::ConfigurationError(
464                "failure_threshold must be >= 1".to_string(),
465            ));
466        }
467        if config.success_threshold == 0 {
468            return Err(BreakerError::ConfigurationError(
469                "success_threshold must be >= 1".to_string(),
470            ));
471        }
472        if config.half_open_probes == 0 {
473            return Err(BreakerError::ConfigurationError(
474                "half_open_probes must be >= 1".to_string(),
475            ));
476        }
477        let window = SlidingWindow::new(config.sliding_window_size);
478        Ok(Self {
479            config,
480            state: CircuitState::Closed {
481                failure_count: 0,
482                success_count: 0,
483            },
484            window,
485            metrics: MetricsAccumulator::default(),
486            events: VecDeque::with_capacity(MAX_EVENT_HISTORY + 1),
487        })
488    }
489
490    /// Try to acquire a call slot.
491    ///
492    /// - **Closed** → always succeeds.
493    /// - **Open** → fails with [`BreakerError::CircuitOpen`] unless
494    ///   `current_ts >= opened_at + retry_after_us`, in which case the circuit
495    ///   automatically transitions to HalfOpen.
496    /// - **HalfOpen** → succeeds if `probe_count < half_open_probes`, otherwise
497    ///   fails with [`BreakerError::MaxProbesExceeded`].
498    pub fn call(&mut self, current_ts: u64) -> Result<CircuitCallGuard, BreakerError> {
499        match &self.state {
500            CircuitState::Closed { .. } => Ok(CircuitCallGuard::new(current_ts)),
501            CircuitState::Open {
502                opened_at,
503                retry_after_us,
504            } => {
505                let threshold = opened_at.saturating_add(*retry_after_us);
506                if current_ts >= threshold {
507                    // Transition to HalfOpen.
508                    let prev_label = self.state.label().to_string();
509                    self.state = CircuitState::HalfOpen {
510                        probe_count: 1,
511                        success_count: 0,
512                    };
513                    self.push_event(CircuitEvent::StateChanged {
514                        from: prev_label,
515                        to: "HalfOpen".to_string(),
516                        at: current_ts,
517                    });
518                    Ok(CircuitCallGuard::new(current_ts))
519                } else {
520                    let remaining = threshold - current_ts;
521                    self.metrics.total_requests += 1;
522                    self.metrics.total_rejections += 1;
523                    Err(BreakerError::CircuitOpen {
524                        retry_after_us: remaining,
525                    })
526                }
527            }
528            CircuitState::HalfOpen {
529                probe_count,
530                success_count,
531            } => {
532                let pc = *probe_count;
533                let sc = *success_count;
534                if pc < self.config.half_open_probes {
535                    self.state = CircuitState::HalfOpen {
536                        probe_count: pc + 1,
537                        success_count: sc,
538                    };
539                    Ok(CircuitCallGuard::new(current_ts))
540                } else {
541                    self.metrics.total_requests += 1;
542                    self.metrics.total_rejections += 1;
543                    Err(BreakerError::MaxProbesExceeded)
544                }
545            }
546        }
547    }
548
549    /// Record the outcome of a permitted call and advance the state machine.
550    ///
551    /// Returns a [`CircuitEvent`] if a notable state transition occurred.
552    ///
553    /// `response_time_us` is the elapsed time in microseconds; it is used only
554    /// for metrics and does not affect the state machine directly.
555    pub fn record_outcome(
556        &mut self,
557        outcome: CircuitOutcome,
558        response_time_us: u64,
559        current_ts: u64,
560    ) -> Option<CircuitEvent> {
561        self.metrics.record_outcome(&outcome, response_time_us);
562
563        match &self.state.clone() {
564            CircuitState::Closed { .. } => self.handle_outcome_closed(outcome, current_ts),
565            CircuitState::HalfOpen {
566                probe_count,
567                success_count,
568            } => {
569                let pc = *probe_count;
570                let sc = *success_count;
571                self.handle_outcome_half_open(outcome, current_ts, pc, sc)
572            }
573            CircuitState::Open { .. } => {
574                // Should not reach here in normal operation (the guard prevents
575                // calls while Open).  We still record the metric above but do
576                // not change state.
577                None
578            }
579        }
580    }
581
582    // ──────────────────────────────────────────────────────────────────────────
583    // Private state-machine helpers
584    // ──────────────────────────────────────────────────────────────────────────
585
586    fn handle_outcome_closed(
587        &mut self,
588        outcome: CircuitOutcome,
589        current_ts: u64,
590    ) -> Option<CircuitEvent> {
591        let is_success = matches!(outcome, CircuitOutcome::Success);
592        self.window.push(is_success);
593
594        let failures_in_window = self.window.failure_count;
595        let successes_in_window = self.window.success_count;
596
597        // Determine whether the failure threshold has been reached.
598        let should_open = failures_in_window >= self.config.failure_threshold
599            && self.window.len() >= self.config.failure_threshold as usize;
600
601        if should_open {
602            let prev = self.state.label().to_string();
603            self.state = CircuitState::Open {
604                opened_at: current_ts,
605                retry_after_us: self.config.open_duration_us,
606            };
607            self.window.reset();
608            let threshold_event = CircuitEvent::ThresholdReached {
609                failures: failures_in_window,
610            };
611            let state_event = CircuitEvent::StateChanged {
612                from: prev,
613                to: "Open".to_string(),
614                at: current_ts,
615            };
616            self.push_event(threshold_event);
617            self.push_event(state_event.clone());
618            Some(state_event)
619        } else {
620            // Stay Closed, update counts.
621            self.state = CircuitState::Closed {
622                failure_count: failures_in_window,
623                success_count: successes_in_window,
624            };
625            None
626        }
627    }
628
629    fn handle_outcome_half_open(
630        &mut self,
631        outcome: CircuitOutcome,
632        current_ts: u64,
633        _probe_count: u32,
634        success_count: u32,
635    ) -> Option<CircuitEvent> {
636        match outcome {
637            CircuitOutcome::Success => {
638                let new_successes = success_count + 1;
639                if new_successes >= self.config.success_threshold {
640                    // Transition to Closed.
641                    let prev = self.state.label().to_string();
642                    self.state = CircuitState::Closed {
643                        failure_count: 0,
644                        success_count: 0,
645                    };
646                    self.window.reset();
647                    let evt = CircuitEvent::RecoverySucceeded;
648                    self.push_event(evt.clone());
649                    let state_evt = CircuitEvent::StateChanged {
650                        from: prev,
651                        to: "Closed".to_string(),
652                        at: current_ts,
653                    };
654                    self.push_event(state_evt);
655                    Some(evt)
656                } else {
657                    // Stay HalfOpen, increment successes.
658                    if let CircuitState::HalfOpen { probe_count, .. } = self.state {
659                        self.state = CircuitState::HalfOpen {
660                            probe_count,
661                            success_count: new_successes,
662                        };
663                    }
664                    None
665                }
666            }
667            CircuitOutcome::Failure(_) | CircuitOutcome::Timeout => {
668                // Any failure in HalfOpen re-opens the circuit.
669                let prev = self.state.label().to_string();
670                self.state = CircuitState::Open {
671                    opened_at: current_ts,
672                    retry_after_us: self.config.open_duration_us,
673                };
674                self.window.reset();
675                let evt = CircuitEvent::RecoveryFailed;
676                self.push_event(evt.clone());
677                let state_evt = CircuitEvent::StateChanged {
678                    from: prev,
679                    to: "Open".to_string(),
680                    at: current_ts,
681                };
682                self.push_event(state_evt);
683                Some(evt)
684            }
685            CircuitOutcome::Rejected => None,
686        }
687    }
688
689    // ──────────────────────────────────────────────────────────────────────────
690    // Admin / test controls
691    // ──────────────────────────────────────────────────────────────────────────
692
693    /// Force the circuit into the Open state regardless of current state.
694    pub fn force_open(&mut self, current_ts: u64) {
695        let prev = self.state.label().to_string();
696        self.state = CircuitState::Open {
697            opened_at: current_ts,
698            retry_after_us: self.config.open_duration_us,
699        };
700        self.window.reset();
701        self.push_event(CircuitEvent::StateChanged {
702            from: prev,
703            to: "Open".to_string(),
704            at: current_ts,
705        });
706    }
707
708    /// Force the circuit into the Closed state regardless of current state.
709    pub fn force_close(&mut self) {
710        let prev = self.state.label().to_string();
711        self.state = CircuitState::Closed {
712            failure_count: 0,
713            success_count: 0,
714        };
715        self.window.reset();
716        self.push_event(CircuitEvent::StateChanged {
717            from: prev,
718            to: "Closed".to_string(),
719            at: 0,
720        });
721    }
722
723    // ──────────────────────────────────────────────────────────────────────────
724    // Accessors
725    // ──────────────────────────────────────────────────────────────────────────
726
727    /// Return a reference to the current circuit state.
728    pub fn state(&self) -> &CircuitState {
729        &self.state
730    }
731
732    /// Reset all accumulated metrics (success/failure/rejection counters and
733    /// response-time average).  The state machine and event history are
734    /// unaffected.
735    pub fn reset_metrics(&mut self) {
736        self.metrics.reset();
737    }
738
739    /// Return a point-in-time metrics snapshot.
740    pub fn metrics(&self, _current_ts: u64) -> CircuitMetrics {
741        let window_success_rate = self.window.success_rate();
742        let window_failure_rate = self.window.failure_rate();
743        CircuitMetrics {
744            success_rate: window_success_rate,
745            failure_rate: window_failure_rate,
746            rejection_rate: self.metrics.rejection_rate(),
747            avg_response_time_us: self.metrics.avg_response_time_us(),
748            total_requests: self.metrics.total_requests,
749            current_state: self.state.label().to_string(),
750        }
751    }
752
753    /// Return the last up-to-50 circuit events.
754    pub fn event_history(&self) -> Vec<CircuitEvent> {
755        self.events.iter().cloned().collect()
756    }
757
758    // ──────────────────────────────────────────────────────────────────────────
759    // Internal helpers
760    // ──────────────────────────────────────────────────────────────────────────
761
762    fn push_event(&mut self, event: CircuitEvent) {
763        if self.events.len() >= MAX_EVENT_HISTORY {
764            self.events.pop_front();
765        }
766        self.events.push_back(event);
767    }
768}
769
770// ─────────────────────────────────────────────────────────────────────────────
771// Tests
772// ─────────────────────────────────────────────────────────────────────────────
773
774#[cfg(test)]
775mod tests {
776    use super::*;
777
778    // ── Helpers ───────────────────────────────────────────────────────────────
779
780    /// Build a breaker with small thresholds to make tests concise.
781    fn make_breaker() -> NetworkCircuitBreaker {
782        let cfg = CircuitConfig {
783            failure_threshold: 3,
784            success_threshold: 2,
785            half_open_probes: 2,
786            open_duration_us: 10_000, // 10 ms
787            timeout_us: 5_000,
788            sliding_window_size: 5,
789        };
790        NetworkCircuitBreaker::new(cfg).expect("valid config")
791    }
792
793    /// Drive `n` failures through the breaker starting at `ts`.
794    fn inject_failures(b: &mut NetworkCircuitBreaker, n: u32, ts: &mut u64) {
795        for _ in 0..n {
796            let mut g = b.call(*ts).expect("call should be permitted");
797            b.record_outcome(CircuitOutcome::Failure("err".into()), 100, *ts);
798            g.mark_recorded();
799            *ts += 1;
800        }
801    }
802
803    /// Drive `n` successes through the breaker starting at `ts`.
804    fn inject_successes(b: &mut NetworkCircuitBreaker, n: u32, ts: &mut u64) {
805        for _ in 0..n {
806            let mut g = b.call(*ts).expect("call should be permitted");
807            b.record_outcome(CircuitOutcome::Success, 100, *ts);
808            g.mark_recorded();
809            *ts += 1;
810        }
811    }
812
813    // ── Construction ──────────────────────────────────────────────────────────
814
815    #[test]
816    fn test_new_starts_closed() {
817        let b = make_breaker();
818        assert!(matches!(b.state(), CircuitState::Closed { .. }));
819    }
820
821    #[test]
822    fn test_new_invalid_window_size() {
823        let cfg = CircuitConfig {
824            sliding_window_size: 0,
825            ..CircuitConfig::default()
826        };
827        assert!(matches!(
828            NetworkCircuitBreaker::new(cfg),
829            Err(BreakerError::ConfigurationError(_))
830        ));
831    }
832
833    #[test]
834    fn test_new_invalid_failure_threshold() {
835        let cfg = CircuitConfig {
836            failure_threshold: 0,
837            ..CircuitConfig::default()
838        };
839        assert!(matches!(
840            NetworkCircuitBreaker::new(cfg),
841            Err(BreakerError::ConfigurationError(_))
842        ));
843    }
844
845    #[test]
846    fn test_new_invalid_success_threshold() {
847        let cfg = CircuitConfig {
848            success_threshold: 0,
849            ..CircuitConfig::default()
850        };
851        assert!(matches!(
852            NetworkCircuitBreaker::new(cfg),
853            Err(BreakerError::ConfigurationError(_))
854        ));
855    }
856
857    #[test]
858    fn test_new_invalid_half_open_probes() {
859        let cfg = CircuitConfig {
860            half_open_probes: 0,
861            ..CircuitConfig::default()
862        };
863        assert!(matches!(
864            NetworkCircuitBreaker::new(cfg),
865            Err(BreakerError::ConfigurationError(_))
866        ));
867    }
868
869    // ── Closed state ──────────────────────────────────────────────────────────
870
871    #[test]
872    fn test_closed_allows_calls() {
873        let mut b = make_breaker();
874        let g = b.call(1000);
875        assert!(g.is_ok());
876    }
877
878    #[test]
879    fn test_closed_stays_closed_on_success() {
880        let mut b = make_breaker();
881        let mut ts = 1000u64;
882        inject_successes(&mut b, 5, &mut ts);
883        assert!(matches!(b.state(), CircuitState::Closed { .. }));
884    }
885
886    #[test]
887    fn test_closed_failure_count_increments() {
888        let mut b = make_breaker();
889        let mut ts = 1000u64;
890        inject_failures(&mut b, 2, &mut ts);
891        match b.state() {
892            CircuitState::Closed { failure_count, .. } => assert_eq!(*failure_count, 2),
893            _ => panic!("expected Closed"),
894        }
895    }
896
897    // ── Closed → Open transition ───────────────────────────────────────────────
898
899    #[test]
900    fn test_closed_to_open_on_threshold() {
901        let mut b = make_breaker();
902        let mut ts = 1000u64;
903        inject_failures(&mut b, 3, &mut ts);
904        assert!(matches!(b.state(), CircuitState::Open { .. }));
905    }
906
907    #[test]
908    fn test_open_emits_threshold_event() {
909        let mut b = make_breaker();
910        let mut ts = 1000u64;
911        inject_failures(&mut b, 3, &mut ts);
912        let hist = b.event_history();
913        let has_threshold = hist
914            .iter()
915            .any(|e| matches!(e, CircuitEvent::ThresholdReached { .. }));
916        assert!(has_threshold, "expected ThresholdReached event");
917    }
918
919    #[test]
920    fn test_open_emits_state_changed_event() {
921        let mut b = make_breaker();
922        let mut ts = 1000u64;
923        inject_failures(&mut b, 3, &mut ts);
924        let hist = b.event_history();
925        let has_state_change = hist
926            .iter()
927            .any(|e| matches!(e, CircuitEvent::StateChanged { to, .. } if to == "Open"));
928        assert!(has_state_change);
929    }
930
931    // ── Open state ────────────────────────────────────────────────────────────
932
933    #[test]
934    fn test_open_rejects_calls_before_timeout() {
935        let mut b = make_breaker();
936        let mut ts = 1000u64;
937        inject_failures(&mut b, 3, &mut ts);
938        // Attempt before the recovery window expires.
939        let err = b.call(ts + 1).unwrap_err();
940        assert!(matches!(err, BreakerError::CircuitOpen { .. }));
941    }
942
943    #[test]
944    fn test_open_retry_after_reported_correctly() {
945        let mut b = make_breaker();
946        let mut ts = 1000u64;
947        inject_failures(&mut b, 3, &mut ts);
948        let call_ts = ts + 1;
949        match b.call(call_ts).unwrap_err() {
950            BreakerError::CircuitOpen { retry_after_us } => {
951                // opened_at = ts - 1 (last failure ts), open_duration = 10_000
952                // remaining = open_duration - elapsed
953                assert!(retry_after_us > 0);
954                assert!(retry_after_us <= 10_000);
955            }
956            other => panic!("unexpected error: {:?}", other),
957        }
958    }
959
960    #[test]
961    fn test_open_increments_rejection_counter() {
962        let mut b = make_breaker();
963        let mut ts = 1000u64;
964        inject_failures(&mut b, 3, &mut ts);
965        let _ = b.call(ts + 1); // rejected
966        let m = b.metrics(ts + 1);
967        assert!(m.rejection_rate > 0.0);
968    }
969
970    // ── Open → HalfOpen transition ────────────────────────────────────────────
971
972    #[test]
973    fn test_open_to_half_open_after_timeout() {
974        let mut b = make_breaker();
975        let mut ts = 1000u64;
976        inject_failures(&mut b, 3, &mut ts);
977        // Advance past open_duration_us.
978        let recovery_ts = ts + 20_000;
979        let g = b.call(recovery_ts);
980        assert!(g.is_ok(), "should transition to HalfOpen");
981        assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
982    }
983
984    #[test]
985    fn test_open_to_half_open_emits_event() {
986        let mut b = make_breaker();
987        let mut ts = 1000u64;
988        inject_failures(&mut b, 3, &mut ts);
989        let _ = b.call(ts + 20_000);
990        let hist = b.event_history();
991        let has = hist
992            .iter()
993            .any(|e| matches!(e, CircuitEvent::StateChanged { to, .. } if to == "HalfOpen"));
994        assert!(has);
995    }
996
997    // ── HalfOpen state ────────────────────────────────────────────────────────
998
999    #[test]
1000    fn test_half_open_allows_limited_probes() {
1001        let mut b = make_breaker();
1002        let mut ts = 1000u64;
1003        inject_failures(&mut b, 3, &mut ts);
1004        let recovery_ts = ts + 20_000;
1005        // First probe (already consumed by the call() that triggered HalfOpen).
1006        // Second probe:
1007        let g2 = b.call(recovery_ts + 1);
1008        assert!(g2.is_ok(), "second probe should be allowed");
1009    }
1010
1011    #[test]
1012    fn test_half_open_rejects_excess_probes() {
1013        let mut b = make_breaker();
1014        let mut ts = 1000u64;
1015        inject_failures(&mut b, 3, &mut ts);
1016        let rt = ts + 20_000;
1017        let _ = b.call(rt); // probe 1 (triggers transition)
1018        let _ = b.call(rt + 1); // probe 2
1019        let err = b.call(rt + 2).unwrap_err();
1020        assert!(matches!(err, BreakerError::MaxProbesExceeded));
1021    }
1022
1023    // ── HalfOpen → Closed (recovery success) ─────────────────────────────────
1024
1025    #[test]
1026    fn test_half_open_to_closed_on_success() {
1027        let mut b = make_breaker();
1028        let mut ts = 1000u64;
1029        inject_failures(&mut b, 3, &mut ts);
1030        let rt = ts + 20_000;
1031        // Trigger transition to HalfOpen.
1032        let mut g = b.call(rt).expect("first probe");
1033        b.record_outcome(CircuitOutcome::Success, 50, rt);
1034        g.mark_recorded();
1035        // success_threshold = 2; second success should close.
1036        let mut g2 = b.call(rt + 1).expect("second probe");
1037        let evt = b.record_outcome(CircuitOutcome::Success, 50, rt + 1);
1038        g2.mark_recorded();
1039        assert!(
1040            matches!(evt, Some(CircuitEvent::RecoverySucceeded)),
1041            "expected RecoverySucceeded"
1042        );
1043        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1044    }
1045
1046    #[test]
1047    fn test_recovery_succeeded_event_in_history() {
1048        let mut b = make_breaker();
1049        let mut ts = 1000u64;
1050        inject_failures(&mut b, 3, &mut ts);
1051        let rt = ts + 20_000;
1052        let mut g = b.call(rt).expect("probe 1");
1053        b.record_outcome(CircuitOutcome::Success, 50, rt);
1054        g.mark_recorded();
1055        let mut g2 = b.call(rt + 1).expect("probe 2");
1056        b.record_outcome(CircuitOutcome::Success, 50, rt + 1);
1057        g2.mark_recorded();
1058        let hist = b.event_history();
1059        let has = hist
1060            .iter()
1061            .any(|e| matches!(e, CircuitEvent::RecoverySucceeded));
1062        assert!(has);
1063    }
1064
1065    // ── HalfOpen → Open (recovery failure) ───────────────────────────────────
1066
1067    #[test]
1068    fn test_half_open_to_open_on_failure() {
1069        let mut b = make_breaker();
1070        let mut ts = 1000u64;
1071        inject_failures(&mut b, 3, &mut ts);
1072        let rt = ts + 20_000;
1073        let mut g = b.call(rt).expect("probe");
1074        let evt = b.record_outcome(CircuitOutcome::Failure("boom".into()), 200, rt);
1075        g.mark_recorded();
1076        assert!(matches!(evt, Some(CircuitEvent::RecoveryFailed)));
1077        assert!(matches!(b.state(), CircuitState::Open { .. }));
1078    }
1079
1080    #[test]
1081    fn test_half_open_timeout_reopens() {
1082        let mut b = make_breaker();
1083        let mut ts = 1000u64;
1084        inject_failures(&mut b, 3, &mut ts);
1085        let rt = ts + 20_000;
1086        let mut g = b.call(rt).expect("probe");
1087        let evt = b.record_outcome(CircuitOutcome::Timeout, 9_999, rt);
1088        g.mark_recorded();
1089        assert!(matches!(evt, Some(CircuitEvent::RecoveryFailed)));
1090        assert!(matches!(b.state(), CircuitState::Open { .. }));
1091    }
1092
1093    #[test]
1094    fn test_recovery_failed_event_in_history() {
1095        let mut b = make_breaker();
1096        let mut ts = 1000u64;
1097        inject_failures(&mut b, 3, &mut ts);
1098        let rt = ts + 20_000;
1099        let mut g = b.call(rt).expect("probe");
1100        b.record_outcome(CircuitOutcome::Failure("x".into()), 10, rt);
1101        g.mark_recorded();
1102        let hist = b.event_history();
1103        assert!(hist
1104            .iter()
1105            .any(|e| matches!(e, CircuitEvent::RecoveryFailed)));
1106    }
1107
1108    // ── Full round-trip ───────────────────────────────────────────────────────
1109
1110    #[test]
1111    fn test_full_state_cycle_closed_open_halfopen_closed() {
1112        let mut b = make_breaker();
1113        let mut ts = 0u64;
1114
1115        // 1. Start Closed.
1116        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1117
1118        // 2. Trip the breaker.
1119        inject_failures(&mut b, 3, &mut ts);
1120        assert!(matches!(b.state(), CircuitState::Open { .. }));
1121
1122        // 3. Wait out the open period.
1123        ts += 20_000;
1124
1125        // 4. First probe → HalfOpen.
1126        let mut g = b.call(ts).expect("probe 1");
1127        b.record_outcome(CircuitOutcome::Success, 10, ts);
1128        g.mark_recorded();
1129        assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
1130
1131        // 5. Second probe → Closed.
1132        ts += 1;
1133        let mut g2 = b.call(ts).expect("probe 2");
1134        b.record_outcome(CircuitOutcome::Success, 10, ts);
1135        g2.mark_recorded();
1136        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1137    }
1138
1139    #[test]
1140    fn test_multiple_trip_recover_cycles() {
1141        let mut b = make_breaker();
1142        let mut ts = 0u64;
1143
1144        for cycle in 0..3u32 {
1145            // Trip.
1146            inject_failures(&mut b, 3, &mut ts);
1147            assert!(
1148                matches!(b.state(), CircuitState::Open { .. }),
1149                "cycle {}",
1150                cycle
1151            );
1152
1153            // Recover.
1154            ts += 20_000;
1155            let mut g = b.call(ts).expect("probe 1");
1156            b.record_outcome(CircuitOutcome::Success, 10, ts);
1157            g.mark_recorded();
1158            ts += 1;
1159            let mut g2 = b.call(ts).expect("probe 2");
1160            b.record_outcome(CircuitOutcome::Success, 10, ts);
1161            g2.mark_recorded();
1162            assert!(
1163                matches!(b.state(), CircuitState::Closed { .. }),
1164                "cycle {} after recovery",
1165                cycle
1166            );
1167            ts += 1;
1168        }
1169    }
1170
1171    // ── Rolling window ────────────────────────────────────────────────────────
1172
1173    #[test]
1174    fn test_sliding_window_evicts_oldest() {
1175        // Window size = 5, threshold = 3.
1176        // Inject 5 successes, then 3 failures → should trip.
1177        let mut b = make_breaker();
1178        let mut ts = 0u64;
1179        inject_successes(&mut b, 5, &mut ts);
1180        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1181        inject_failures(&mut b, 3, &mut ts);
1182        assert!(matches!(b.state(), CircuitState::Open { .. }));
1183    }
1184
1185    #[test]
1186    fn test_sliding_window_does_not_trip_if_failures_old() {
1187        // Window = 5, threshold = 3.
1188        // Inject 2 failures, then 5 successes (evict the 2 failures), then 2 failures.
1189        // The 2 new failures should NOT trip (only 2 in current window, threshold=3).
1190        let mut b = make_breaker();
1191        let mut ts = 0u64;
1192        inject_failures(&mut b, 2, &mut ts);
1193        // Inject enough successes to push the failures out of the window.
1194        inject_successes(&mut b, 5, &mut ts);
1195        // Now inject 2 more failures — should not trip.
1196        inject_failures(&mut b, 2, &mut ts);
1197        assert!(
1198            matches!(b.state(), CircuitState::Closed { .. }),
1199            "window should have evicted old failures"
1200        );
1201    }
1202
1203    #[test]
1204    fn test_sliding_window_failure_rate() {
1205        let mut sw = SlidingWindow::new(4);
1206        sw.push(false);
1207        sw.push(false);
1208        sw.push(true);
1209        sw.push(true);
1210        assert!((sw.failure_rate() - 0.5).abs() < f64::EPSILON);
1211        assert!((sw.success_rate() - 0.5).abs() < f64::EPSILON);
1212    }
1213
1214    #[test]
1215    fn test_sliding_window_evicts_when_full() {
1216        let mut sw = SlidingWindow::new(3);
1217        sw.push(false); // evicted next
1218        sw.push(true);
1219        sw.push(true);
1220        sw.push(true); // evicts first false
1221        assert_eq!(sw.failure_count, 0);
1222        assert_eq!(sw.success_count, 3);
1223    }
1224
1225    #[test]
1226    fn test_sliding_window_empty_failure_rate() {
1227        let sw = SlidingWindow::new(5);
1228        assert!((sw.failure_rate() - 0.0).abs() < f64::EPSILON);
1229        assert!((sw.success_rate() - 1.0).abs() < f64::EPSILON);
1230    }
1231
1232    // ── force_open / force_close ──────────────────────────────────────────────
1233
1234    #[test]
1235    fn test_force_open_from_closed() {
1236        let mut b = make_breaker();
1237        b.force_open(5000);
1238        assert!(matches!(b.state(), CircuitState::Open { .. }));
1239    }
1240
1241    #[test]
1242    fn test_force_close_from_open() {
1243        let mut b = make_breaker();
1244        let mut ts = 0u64;
1245        inject_failures(&mut b, 3, &mut ts);
1246        b.force_close();
1247        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1248    }
1249
1250    #[test]
1251    fn test_force_open_from_half_open() {
1252        let mut b = make_breaker();
1253        let mut ts = 0u64;
1254        inject_failures(&mut b, 3, &mut ts);
1255        let _ = b.call(ts + 20_000);
1256        b.force_open(ts + 25_000);
1257        assert!(matches!(b.state(), CircuitState::Open { .. }));
1258    }
1259
1260    #[test]
1261    fn test_force_open_emits_event() {
1262        let mut b = make_breaker();
1263        b.force_open(999);
1264        let hist = b.event_history();
1265        assert!(hist.iter().any(|e| {
1266            matches!(e, CircuitEvent::StateChanged { to, at, .. } if to == "Open" && *at == 999)
1267        }));
1268    }
1269
1270    #[test]
1271    fn test_force_close_emits_event() {
1272        let mut b = make_breaker();
1273        let mut ts = 0u64;
1274        inject_failures(&mut b, 3, &mut ts);
1275        b.force_close();
1276        let hist = b.event_history();
1277        assert!(hist
1278            .iter()
1279            .any(|e| { matches!(e, CircuitEvent::StateChanged { to, .. } if to == "Closed") }));
1280    }
1281
1282    #[test]
1283    fn test_force_close_resets_window() {
1284        let mut b = make_breaker();
1285        let mut ts = 0u64;
1286        inject_failures(&mut b, 2, &mut ts);
1287        b.force_close();
1288        // After force_close, window is reset → 2 new failures should not trip
1289        // (threshold=3 but window was reset).
1290        inject_failures(&mut b, 2, &mut ts);
1291        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1292    }
1293
1294    // ── Metrics ───────────────────────────────────────────────────────────────
1295
1296    #[test]
1297    fn test_metrics_initial_state() {
1298        let b = make_breaker();
1299        let m = b.metrics(0);
1300        assert_eq!(m.total_requests, 0);
1301        assert_eq!(m.current_state, "Closed");
1302        assert!((m.failure_rate - 0.0).abs() < f64::EPSILON);
1303    }
1304
1305    #[test]
1306    fn test_metrics_success_rate() {
1307        let mut b = make_breaker();
1308        let mut ts = 0u64;
1309        inject_successes(&mut b, 4, &mut ts);
1310        let m = b.metrics(ts);
1311        assert!((m.success_rate - 1.0).abs() < f64::EPSILON);
1312        assert!((m.failure_rate - 0.0).abs() < f64::EPSILON);
1313    }
1314
1315    #[test]
1316    fn test_metrics_failure_rate() {
1317        let mut b = make_breaker();
1318        let mut ts = 0u64;
1319        inject_failures(&mut b, 2, &mut ts);
1320        inject_successes(&mut b, 2, &mut ts);
1321        let m = b.metrics(ts);
1322        assert!((m.failure_rate - 0.5).abs() < f64::EPSILON);
1323    }
1324
1325    #[test]
1326    fn test_metrics_rejection_rate() {
1327        let mut b = make_breaker();
1328        let mut ts = 0u64;
1329        inject_failures(&mut b, 3, &mut ts);
1330        // Now Open; one more rejected call.
1331        let _ = b.call(ts + 1);
1332        // Total accumulator requests = 3 (failures) + 1 (rejection) + 1 (from the rejection itself) = ...
1333        // The rejection counter in MetricsAccumulator is separate from the window.
1334        let m = b.metrics(ts + 1);
1335        assert!(m.rejection_rate > 0.0);
1336    }
1337
1338    #[test]
1339    fn test_metrics_avg_response_time() {
1340        let mut b = make_breaker();
1341        let mut ts = 0u64;
1342        let mut g1 = b.call(ts).expect("call");
1343        b.record_outcome(CircuitOutcome::Success, 100, ts);
1344        g1.mark_recorded();
1345        ts += 1;
1346        let mut g2 = b.call(ts).expect("call");
1347        b.record_outcome(CircuitOutcome::Success, 300, ts);
1348        g2.mark_recorded();
1349        let m = b.metrics(ts);
1350        assert!((m.avg_response_time_us - 200.0).abs() < f64::EPSILON);
1351    }
1352
1353    #[test]
1354    fn test_reset_metrics() {
1355        let mut b = make_breaker();
1356        let mut ts = 0u64;
1357        inject_successes(&mut b, 3, &mut ts);
1358        b.reset_metrics();
1359        let m = b.metrics(ts);
1360        assert_eq!(m.total_requests, 0);
1361    }
1362
1363    #[test]
1364    fn test_metrics_current_state_open() {
1365        let mut b = make_breaker();
1366        let mut ts = 0u64;
1367        inject_failures(&mut b, 3, &mut ts);
1368        let m = b.metrics(ts);
1369        assert_eq!(m.current_state, "Open");
1370    }
1371
1372    #[test]
1373    fn test_metrics_current_state_half_open() {
1374        let mut b = make_breaker();
1375        let mut ts = 0u64;
1376        inject_failures(&mut b, 3, &mut ts);
1377        let rt = ts + 20_000;
1378        let mut g = b.call(rt).expect("probe");
1379        b.record_outcome(CircuitOutcome::Success, 10, rt);
1380        g.mark_recorded();
1381        let m = b.metrics(rt);
1382        assert_eq!(m.current_state, "HalfOpen");
1383    }
1384
1385    // ── Event history ─────────────────────────────────────────────────────────
1386
1387    #[test]
1388    fn test_event_history_empty_initially() {
1389        let b = make_breaker();
1390        assert!(b.event_history().is_empty());
1391    }
1392
1393    #[test]
1394    fn test_event_history_capped_at_50() {
1395        // Use a breaker with window=1 and threshold=1 so every failure trips it.
1396        let cfg = CircuitConfig {
1397            failure_threshold: 1,
1398            success_threshold: 1,
1399            half_open_probes: 1,
1400            open_duration_us: 1,
1401            timeout_us: 1_000_000,
1402            sliding_window_size: 1,
1403        };
1404        let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1405        let mut ts = 0u64;
1406
1407        for _ in 0..60u32 {
1408            // Reset to Closed first.
1409            b.force_close();
1410            // Trip it.
1411            let mut g = b.call(ts).expect("call");
1412            b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1413            g.mark_recorded();
1414            ts += 2;
1415            // Recover (open_duration=1µs so immediate).
1416            let mut g2 = b.call(ts).expect("probe");
1417            b.record_outcome(CircuitOutcome::Success, 10, ts);
1418            g2.mark_recorded();
1419            ts += 2;
1420        }
1421
1422        let hist = b.event_history();
1423        assert!(
1424            hist.len() <= 50,
1425            "history must not exceed 50 events, got {}",
1426            hist.len()
1427        );
1428    }
1429
1430    #[test]
1431    fn test_event_history_records_all_transitions() {
1432        let mut b = make_breaker();
1433        let mut ts = 0u64;
1434        inject_failures(&mut b, 3, &mut ts);
1435        ts += 20_000;
1436        let mut g = b.call(ts).expect("probe 1");
1437        b.record_outcome(CircuitOutcome::Success, 10, ts);
1438        g.mark_recorded();
1439        ts += 1;
1440        let mut g2 = b.call(ts).expect("probe 2");
1441        b.record_outcome(CircuitOutcome::Success, 10, ts);
1442        g2.mark_recorded();
1443
1444        let hist = b.event_history();
1445        let labels: Vec<&str> = hist
1446            .iter()
1447            .filter_map(|e| {
1448                if let CircuitEvent::StateChanged { to, .. } = e {
1449                    Some(to.as_str())
1450                } else {
1451                    None
1452                }
1453            })
1454            .collect();
1455
1456        assert!(labels.contains(&"Open"), "should have Open transition");
1457        assert!(
1458            labels.contains(&"HalfOpen"),
1459            "should have HalfOpen transition"
1460        );
1461        assert!(labels.contains(&"Closed"), "should have Closed transition");
1462    }
1463
1464    // ── CircuitCallGuard ──────────────────────────────────────────────────────
1465
1466    #[test]
1467    fn test_guard_mark_recorded() {
1468        let mut g = CircuitCallGuard::new(0);
1469        assert!(!g.is_recorded());
1470        g.mark_recorded();
1471        assert!(g.is_recorded());
1472    }
1473
1474    #[test]
1475    fn test_guard_issued_at() {
1476        let g = CircuitCallGuard::new(42_000);
1477        assert_eq!(g.issued_at, 42_000);
1478    }
1479
1480    #[test]
1481    fn test_guard_drop_without_recording() {
1482        // Should not panic — just drops silently.
1483        {
1484            let _g = CircuitCallGuard::new(100);
1485            // Drop without calling mark_recorded().
1486        }
1487    }
1488
1489    // ── PRNG ──────────────────────────────────────────────────────────────────
1490
1491    #[test]
1492    fn test_xorshift64_deterministic() {
1493        let mut state = 12345u64;
1494        let v1 = xorshift64(&mut state);
1495        let mut state2 = 12345u64;
1496        let v2 = xorshift64(&mut state2);
1497        assert_eq!(v1, v2);
1498    }
1499
1500    #[test]
1501    fn test_xorshift64_non_zero() {
1502        let mut state = 1u64;
1503        for _ in 0..100 {
1504            let v = xorshift64(&mut state);
1505            assert_ne!(v, 0);
1506        }
1507    }
1508
1509    #[test]
1510    fn test_xorshift64_distinct_values() {
1511        let mut state = 999u64;
1512        let a = xorshift64(&mut state);
1513        let b = xorshift64(&mut state);
1514        assert_ne!(a, b);
1515    }
1516
1517    // ── BreakerError display ──────────────────────────────────────────────────
1518
1519    #[test]
1520    fn test_breaker_error_display_circuit_open() {
1521        let e = BreakerError::CircuitOpen {
1522            retry_after_us: 5000,
1523        };
1524        let s = format!("{}", e);
1525        assert!(s.contains("5000"));
1526    }
1527
1528    #[test]
1529    fn test_breaker_error_display_max_probes() {
1530        let e = BreakerError::MaxProbesExceeded;
1531        let s = format!("{}", e);
1532        assert!(!s.is_empty());
1533    }
1534
1535    #[test]
1536    fn test_breaker_error_display_config() {
1537        let e = BreakerError::ConfigurationError("bad value".into());
1538        let s = format!("{}", e);
1539        assert!(s.contains("bad value"));
1540    }
1541
1542    // ── Edge cases ────────────────────────────────────────────────────────────
1543
1544    #[test]
1545    fn test_record_outcome_in_open_state_no_panic() {
1546        let mut b = make_breaker();
1547        let mut ts = 0u64;
1548        inject_failures(&mut b, 3, &mut ts);
1549        // Manually call record_outcome while Open (should be a no-op for state).
1550        let evt = b.record_outcome(CircuitOutcome::Success, 10, ts);
1551        assert!(evt.is_none(), "no state change expected");
1552        assert!(matches!(b.state(), CircuitState::Open { .. }));
1553    }
1554
1555    #[test]
1556    fn test_exact_boundary_open_to_half_open() {
1557        let mut b = make_breaker();
1558        let mut ts = 1000u64;
1559        inject_failures(&mut b, 3, &mut ts);
1560        // opened_at = ts - 1 (last injected failure ts was ts-1).
1561        // open_duration = 10_000
1562        // Exact boundary: opened_at + 10_000.
1563        if let CircuitState::Open {
1564            opened_at,
1565            retry_after_us,
1566        } = b.state().clone()
1567        {
1568            let exact_boundary = opened_at + retry_after_us;
1569            // One microsecond before: still Open.
1570            let err = b.call(exact_boundary - 1).unwrap_err();
1571            assert!(matches!(err, BreakerError::CircuitOpen { .. }));
1572            // Exact boundary: transition to HalfOpen.
1573            let g = b.call(exact_boundary);
1574            assert!(g.is_ok());
1575        } else {
1576            panic!("expected Open state");
1577        }
1578    }
1579
1580    #[test]
1581    fn test_half_open_probe_count_increments_on_call() {
1582        let mut b = make_breaker();
1583        let mut ts = 0u64;
1584        inject_failures(&mut b, 3, &mut ts);
1585        let rt = ts + 20_000;
1586        let mut g = b.call(rt).expect("probe 1");
1587        // At this point state should be HalfOpen with probe_count=1.
1588        match b.state() {
1589            CircuitState::HalfOpen { probe_count, .. } => assert_eq!(*probe_count, 1),
1590            _ => panic!("expected HalfOpen"),
1591        }
1592        g.mark_recorded();
1593        // Probe 2: probe_count becomes 2 (before call() increments).
1594        let _g2 = b.call(rt + 1).expect("probe 2");
1595        match b.state() {
1596            CircuitState::HalfOpen { probe_count, .. } => assert_eq!(*probe_count, 2),
1597            _ => panic!("expected HalfOpen"),
1598        }
1599    }
1600
1601    #[test]
1602    fn test_no_false_trip_on_mixed_outcomes() {
1603        // Use a larger window (10) and threshold=4 so that alternating S/F pairs
1604        // (window contains at most 5 failures out of 10) stay well below the
1605        // threshold.  We verify that 10 pairs do not permanently latch the circuit.
1606        let cfg = CircuitConfig {
1607            failure_threshold: 6, // requires 6/10 = 60% failure rate to trip
1608            success_threshold: 2,
1609            half_open_probes: 2,
1610            open_duration_us: 10_000,
1611            timeout_us: 5_000,
1612            sliding_window_size: 10,
1613        };
1614        let mut b = NetworkCircuitBreaker::new(cfg).expect("valid config");
1615        let mut ts = 0u64;
1616        // 10 pairs of (Success, Failure) = 10 successes and 10 failures but window
1617        // only holds 10 entries → the window will always have exactly 5 failures.
1618        // 5 < 6 (failure_threshold), so the circuit must remain Closed.
1619        for _ in 0..10u32 {
1620            let mut g = b.call(ts).expect("call");
1621            b.record_outcome(CircuitOutcome::Success, 50, ts);
1622            g.mark_recorded();
1623            ts += 1;
1624            let mut g2 = b.call(ts).expect("call");
1625            b.record_outcome(CircuitOutcome::Failure("transient".into()), 100, ts);
1626            g2.mark_recorded();
1627            ts += 1;
1628        }
1629        assert!(
1630            matches!(b.state(), CircuitState::Closed { .. }),
1631            "circuit should remain Closed with 50% failure rate below threshold"
1632        );
1633    }
1634
1635    #[test]
1636    fn test_rejected_outcome_does_not_affect_window() {
1637        let mut b = make_breaker();
1638        let mut ts = 0u64;
1639        inject_failures(&mut b, 3, &mut ts);
1640        // Multiple rejections.
1641        let _ = b.call(ts + 1);
1642        let _ = b.call(ts + 2);
1643        // After force_close, window should be clean.
1644        b.force_close();
1645        // 2 failures should not trip (threshold=3).
1646        inject_failures(&mut b, 2, &mut ts);
1647        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1648    }
1649
1650    #[test]
1651    fn test_ncb_aliases_are_correct_types() {
1652        // Just verify the type aliases compile and refer to the right types.
1653        let cfg: NcbCircuitConfig = CircuitConfig::default();
1654        let b = NetworkCircuitBreaker::new(cfg).expect("valid");
1655        let state_ref: &NcbCircuitState = b.state();
1656        assert!(matches!(state_ref, CircuitState::Closed { .. }));
1657    }
1658
1659    #[test]
1660    fn test_large_window_partial_fill() {
1661        let cfg = CircuitConfig {
1662            failure_threshold: 3,
1663            success_threshold: 2,
1664            half_open_probes: 2,
1665            open_duration_us: 10_000,
1666            timeout_us: 5_000,
1667            sliding_window_size: 100,
1668        };
1669        let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1670        let mut ts = 0u64;
1671        // With only 2 failures in a window of 100, should not trip.
1672        inject_failures(&mut b, 2, &mut ts);
1673        assert!(matches!(b.state(), CircuitState::Closed { .. }));
1674        // 3rd failure should trip (failure_threshold = 3).
1675        let mut g = b.call(ts).expect("call before trip");
1676        b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1677        g.mark_recorded();
1678        let _ts_next = ts + 1; // advance timestamp (unused after this point)
1679        assert!(
1680            matches!(b.state(), CircuitState::Open { .. }),
1681            "circuit should be Open after 3rd failure"
1682        );
1683    }
1684
1685    #[test]
1686    fn test_metrics_accumulator_timeout_counted_as_failure() {
1687        let mut acc = MetricsAccumulator::default();
1688        acc.record_outcome(&CircuitOutcome::Timeout, 999);
1689        assert_eq!(acc.total_timeouts, 1);
1690        assert_eq!(acc.total_failures, 1);
1691        assert_eq!(acc.total_requests, 1);
1692    }
1693
1694    #[test]
1695    fn test_metrics_accumulator_rejected_not_in_response_time() {
1696        let mut acc = MetricsAccumulator::default();
1697        acc.record_outcome(&CircuitOutcome::Rejected, 0);
1698        assert_eq!(acc.response_time_samples, 0);
1699        assert_eq!(acc.total_rejections, 1);
1700    }
1701
1702    #[test]
1703    fn test_open_duration_zero_immediate_halfopen() {
1704        let cfg = CircuitConfig {
1705            failure_threshold: 1,
1706            success_threshold: 1,
1707            half_open_probes: 1,
1708            open_duration_us: 0,
1709            timeout_us: 1_000_000,
1710            sliding_window_size: 1,
1711        };
1712        let mut b = NetworkCircuitBreaker::new(cfg).expect("valid");
1713        let ts = 0u64;
1714        let mut g = b.call(ts).expect("call");
1715        b.record_outcome(CircuitOutcome::Failure("x".into()), 10, ts);
1716        g.mark_recorded();
1717        assert!(matches!(b.state(), CircuitState::Open { .. }));
1718        // With open_duration=0, same timestamp should trigger HalfOpen.
1719        let g2 = b.call(ts);
1720        assert!(g2.is_ok());
1721        assert!(matches!(b.state(), CircuitState::HalfOpen { .. }));
1722    }
1723}