use std::num::NonZeroUsize;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use freenet_stdlib::prelude::ContractKey;
use lru::LruCache;
use tokio::time::Instant;
use crate::ring::interest::PeerKey;
pub(crate) const QUARANTINE_THRESHOLD: u32 = 5;
pub(crate) const EDGE_CAPACITY: usize = 32_768;
pub(crate) const LONG_GAP_THRESHOLD: Duration = Duration::from_secs(60 * 60);
pub(crate) const LADDER_RUNGS: [u32; 8] = [1, 2, 3, 4, 5, 8, 16, 32];
pub(crate) const LADDER_LEN: usize = LADDER_RUNGS.len();
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum OutcomeEvidence {
Verdict,
ProbeBudgetExhausted,
ProbeUnavailable,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct EdgeState {
pending_since: Option<Instant>,
consecutive_futile: u32,
ladder_recorded: u8,
at_threshold: bool,
}
impl EdgeState {
fn has_streak(&self) -> bool {
self.consecutive_futile > 0
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct FutileRepairSnapshot {
pub(crate) attempts: u64,
pub(crate) futile: u64,
pub(crate) productive: u64,
pub(crate) observations_unpaired: u64,
pub(crate) attempts_superseded: u64,
pub(crate) attempts_discarded: u64,
pub(crate) outcomes_probe_budget_exhausted: u64,
pub(crate) outcomes_probe_unavailable: u64,
pub(crate) outcomes_after_long_gap: u64,
pub(crate) would_quarantine: u64,
pub(crate) edges_at_threshold: u64,
pub(crate) tracked_edges: u64,
pub(crate) evictions: u64,
pub(crate) evictions_losing_streak: u64,
pub(crate) ladder: [u64; LADDER_LEN],
}
pub(crate) const SNAPSHOT_SCALARS: usize = 14;
impl FutileRepairSnapshot {
pub(crate) fn to_row(self) -> [u64; SNAPSHOT_SCALARS] {
[
self.attempts,
self.futile,
self.productive,
self.observations_unpaired,
self.attempts_superseded,
self.attempts_discarded,
self.outcomes_probe_budget_exhausted,
self.outcomes_probe_unavailable,
self.outcomes_after_long_gap,
self.would_quarantine,
self.edges_at_threshold,
self.tracked_edges,
self.evictions,
self.evictions_losing_streak,
]
}
}
struct Metrics {
attempts: AtomicU64,
futile: AtomicU64,
productive: AtomicU64,
observations_unpaired: AtomicU64,
attempts_superseded: AtomicU64,
attempts_discarded: AtomicU64,
outcomes_probe_budget_exhausted: AtomicU64,
outcomes_probe_unavailable: AtomicU64,
outcomes_after_long_gap: AtomicU64,
would_quarantine: AtomicU64,
evictions: AtomicU64,
evictions_losing_streak: AtomicU64,
ladder: [AtomicU64; LADDER_LEN],
}
impl Metrics {
fn new() -> Self {
Self {
attempts: AtomicU64::new(0),
futile: AtomicU64::new(0),
productive: AtomicU64::new(0),
observations_unpaired: AtomicU64::new(0),
attempts_superseded: AtomicU64::new(0),
attempts_discarded: AtomicU64::new(0),
outcomes_probe_budget_exhausted: AtomicU64::new(0),
outcomes_probe_unavailable: AtomicU64::new(0),
outcomes_after_long_gap: AtomicU64::new(0),
would_quarantine: AtomicU64::new(0),
evictions: AtomicU64::new(0),
evictions_losing_streak: AtomicU64::new(0),
ladder: std::array::from_fn(|_| AtomicU64::new(0)),
}
}
}
pub(crate) struct FutileRepairDetector {
edges: Mutex<LruCache<(ContractKey, PeerKey), EdgeState>>,
edges_at_threshold: AtomicU64,
metrics: Metrics,
}
impl Default for FutileRepairDetector {
fn default() -> Self {
Self::new()
}
}
impl FutileRepairDetector {
pub(crate) fn new() -> Self {
Self::with_capacity(EDGE_CAPACITY)
}
pub(crate) fn with_capacity(capacity: usize) -> Self {
Self {
edges: Mutex::new(LruCache::new(
NonZeroUsize::new(capacity).expect("futile-repair capacity must be > 0"),
)),
edges_at_threshold: AtomicU64::new(0),
metrics: Metrics::new(),
}
}
pub(crate) fn record_repair_attempt(
&self,
contract: &ContractKey,
peer: &PeerKey,
now: Instant,
) {
self.metrics.attempts.fetch_add(1, Ordering::Relaxed);
let mut edges = self.lock();
let key = (*contract, peer.clone());
if let Some(state) = edges.get_mut(&key) {
if state.pending_since.is_some() {
self.metrics
.attempts_superseded
.fetch_add(1, Ordering::Relaxed);
}
state.pending_since = Some(now);
return;
}
let state = EdgeState {
pending_since: Some(now),
..EdgeState::default()
};
if let Some((displaced_key, displaced)) = edges.push(key.clone(), state)
&& displaced_key != key
{
self.note_eviction(displaced);
}
}
pub(crate) fn record_repair_outcome(
&self,
contract: &ContractKey,
peer: &PeerKey,
converged: bool,
evidence: OutcomeEvidence,
now: Instant,
) {
match evidence {
OutcomeEvidence::ProbeBudgetExhausted => {
self.metrics
.outcomes_probe_budget_exhausted
.fetch_add(1, Ordering::Relaxed);
return;
}
OutcomeEvidence::ProbeUnavailable => {
self.metrics
.outcomes_probe_unavailable
.fetch_add(1, Ordering::Relaxed);
return;
}
OutcomeEvidence::Verdict => {}
}
let mut edges = self.lock();
let key = (*contract, peer.clone());
let Some(state) = edges.get_mut(&key) else {
self.metrics
.observations_unpaired
.fetch_add(1, Ordering::Relaxed);
return;
};
let Some(pending_since) = state.pending_since.take() else {
self.metrics
.observations_unpaired
.fetch_add(1, Ordering::Relaxed);
return;
};
if now.saturating_duration_since(pending_since) > LONG_GAP_THRESHOLD {
self.metrics
.outcomes_after_long_gap
.fetch_add(1, Ordering::Relaxed);
}
if converged {
self.metrics.productive.fetch_add(1, Ordering::Relaxed);
state.consecutive_futile = 0;
state.ladder_recorded = 0;
if std::mem::take(&mut state.at_threshold) {
self.edges_at_threshold.fetch_sub(1, Ordering::Relaxed);
}
return;
}
self.metrics.futile.fetch_add(1, Ordering::Relaxed);
state.consecutive_futile = state.consecutive_futile.saturating_add(1);
let streak = state.consecutive_futile;
while (state.ladder_recorded as usize) < LADDER_LEN
&& streak >= LADDER_RUNGS[state.ladder_recorded as usize]
{
self.metrics.ladder[state.ladder_recorded as usize].fetch_add(1, Ordering::Relaxed);
state.ladder_recorded += 1;
}
if !state.at_threshold && streak >= QUARANTINE_THRESHOLD {
state.at_threshold = true;
self.metrics
.would_quarantine
.fetch_add(1, Ordering::Relaxed);
self.edges_at_threshold.fetch_add(1, Ordering::Relaxed);
}
}
pub(crate) fn discard_peer_attempts<'a>(
&self,
peer: &PeerKey,
contracts: impl IntoIterator<Item = &'a ContractKey>,
) {
let mut edges = self.lock();
for contract in contracts {
let Some(state) = edges.pop(&(*contract, peer.clone())) else {
continue;
};
if state.pending_since.is_some() {
self.metrics
.attempts_discarded
.fetch_add(1, Ordering::Relaxed);
}
if state.at_threshold {
self.edges_at_threshold.fetch_sub(1, Ordering::Relaxed);
}
}
}
pub(crate) fn snapshot(&self) -> FutileRepairSnapshot {
let load = |value: &AtomicU64| value.load(Ordering::Relaxed);
let tracked_edges = self.lock().len() as u64;
FutileRepairSnapshot {
attempts: load(&self.metrics.attempts),
futile: load(&self.metrics.futile),
productive: load(&self.metrics.productive),
observations_unpaired: load(&self.metrics.observations_unpaired),
attempts_superseded: load(&self.metrics.attempts_superseded),
attempts_discarded: load(&self.metrics.attempts_discarded),
outcomes_probe_budget_exhausted: load(&self.metrics.outcomes_probe_budget_exhausted),
outcomes_probe_unavailable: load(&self.metrics.outcomes_probe_unavailable),
outcomes_after_long_gap: load(&self.metrics.outcomes_after_long_gap),
would_quarantine: load(&self.metrics.would_quarantine),
edges_at_threshold: load(&self.edges_at_threshold),
tracked_edges,
evictions: load(&self.metrics.evictions),
evictions_losing_streak: load(&self.metrics.evictions_losing_streak),
ladder: std::array::from_fn(|i| load(&self.metrics.ladder[i])),
}
}
fn note_eviction(&self, evicted: EdgeState) {
self.metrics.evictions.fetch_add(1, Ordering::Relaxed);
if evicted.has_streak() {
self.metrics
.evictions_losing_streak
.fetch_add(1, Ordering::Relaxed);
}
if evicted.at_threshold {
self.edges_at_threshold.fetch_sub(1, Ordering::Relaxed);
}
}
fn lock(&self) -> std::sync::MutexGuard<'_, LruCache<(ContractKey, PeerKey), EdgeState>> {
self.edges.lock().unwrap_or_else(|poisoned| {
poisoned.into_inner()
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use freenet_stdlib::prelude::{CodeHash, ContractInstanceId};
fn contract(seed: u8) -> ContractKey {
ContractKey::from_id_and_code(
ContractInstanceId::new([seed; 32]),
CodeHash::new([seed.wrapping_add(1); 32]),
)
}
fn peer(seed: u8) -> PeerKey {
let mut bytes = [0u8; 32];
bytes[0] = seed;
PeerKey(crate::transport::TransportPublicKey::from_bytes(bytes))
}
fn t0() -> Instant {
Instant::now()
}
#[test]
fn futility_counts_the_outcome_not_the_attempt() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let converging = contract(1);
let stuck = contract(2);
let p = peer(1);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&converging, &p, now);
detector.record_repair_outcome(&converging, &p, true, OutcomeEvidence::Verdict, now);
}
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&stuck, &p, now);
detector.record_repair_outcome(&stuck, &p, false, OutcomeEvidence::Verdict, now);
}
let snap = detector.snapshot();
assert_eq!(
snap.attempts,
u64::from(QUARANTINE_THRESHOLD) * 2,
"both edges attempted the same number of repairs"
);
assert_eq!(
snap.productive,
u64::from(QUARANTINE_THRESHOLD),
"the converging edge's repairs must all count as productive"
);
assert_eq!(
snap.futile,
u64::from(QUARANTINE_THRESHOLD),
"only the non-convergent edge's repairs are futile"
);
assert_eq!(
snap.would_quarantine, 1,
"exactly one edge reached the threshold — a detector counting \
attempts rather than outcomes would report two"
);
assert_eq!(
snap.edges_at_threshold, 1,
"the converging edge must never be at the threshold"
);
}
#[test]
fn one_productive_repair_resets_the_streak() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(3);
let p = peer(1);
for _ in 0..(QUARANTINE_THRESHOLD - 1) {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, true, OutcomeEvidence::Verdict, now);
assert_eq!(detector.snapshot().would_quarantine, 0);
for _ in 0..(QUARANTINE_THRESHOLD - 1) {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
assert_eq!(
detector.snapshot().would_quarantine,
0,
"the streak must restart at zero after a productive repair"
);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
let snap = detector.snapshot();
assert_eq!(snap.would_quarantine, 1);
assert_eq!(
snap.productive, 1,
"the one landed repair stays visible as productive"
);
}
#[test]
fn observations_without_an_attempt_are_unpaired() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(4);
let p = peer(1);
for _ in 0..10 {
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
let snap = detector.snapshot();
assert_eq!(snap.observations_unpaired, 10);
assert_eq!(
snap.futile, 0,
"no repair was attempted, so none was futile"
);
assert_eq!(snap.would_quarantine, 0);
assert_eq!(snap.tracked_edges, 0, "observations must not create edges");
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
let snap = detector.snapshot();
assert_eq!(snap.futile, 1, "one attempt, one futile outcome");
assert_eq!(snap.observations_unpaired, 11);
}
#[test]
fn a_slow_rotation_still_settles_the_attempt() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(5);
let p = peer(1);
let ten_hours = Duration::from_secs(10 * 60 * 60);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(
&key,
&p,
false,
OutcomeEvidence::Verdict,
now + ten_hours,
);
}
let snap = detector.snapshot();
assert_eq!(
snap.futile,
u64::from(QUARANTINE_THRESHOLD),
"an outcome observed a rotation period later must still settle its \
attempt — a wall-clock expiry here blinds the detector on exactly \
the heavy-summary links it exists to find"
);
assert_eq!(
snap.would_quarantine, 1,
"the streak must be able to reach the threshold on a slow link"
);
assert_eq!(
snap.outcomes_after_long_gap,
u64::from(QUARANTINE_THRESHOLD),
"long-gap settlements are classified, but must stay separately \
visible so the field data can say how much of the headline they \
carry"
);
assert_eq!(
snap.attempts_discarded, 0,
"nothing was torn down, so nothing was discarded"
);
}
#[test]
fn a_prompt_settlement_is_not_flagged_as_a_long_gap() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(6);
let p = peer(1);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(
&key,
&p,
false,
OutcomeEvidence::Verdict,
now + LONG_GAP_THRESHOLD,
);
let snap = detector.snapshot();
assert_eq!(snap.futile, 1);
assert_eq!(
snap.outcomes_after_long_gap, 0,
"at the threshold exactly is not past it"
);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(
&key,
&p,
false,
OutcomeEvidence::Verdict,
now + LONG_GAP_THRESHOLD + Duration::from_secs(1),
);
assert_eq!(detector.snapshot().outcomes_after_long_gap, 1);
}
#[test]
fn peer_teardown_discards_the_outstanding_attempt() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(7);
let p = peer(1);
let other = peer(2);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_attempt(&key, &other, now);
detector.discard_peer_attempts(&p, [&key]);
let snap = detector.snapshot();
assert_eq!(snap.attempts_discarded, 1);
assert_eq!(
snap.tracked_edges, 1,
"only the departing peer's edge is dropped"
);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
let snap = detector.snapshot();
assert_eq!(
snap.futile, 0,
"a heal to a peer that left must not be settled by a comparison \
made after it came back"
);
assert_eq!(snap.observations_unpaired, 1);
detector.record_repair_outcome(&key, &other, false, OutcomeEvidence::Verdict, now);
assert_eq!(detector.snapshot().futile, 1);
}
#[test]
fn teardown_releases_the_at_threshold_gauge() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(8);
let p = peer(1);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
assert_eq!(detector.snapshot().edges_at_threshold, 1);
detector.discard_peer_attempts(&p, [&key]);
let snap = detector.snapshot();
assert_eq!(snap.edges_at_threshold, 0);
assert_eq!(
snap.would_quarantine, 1,
"the crossing already happened and stays counted"
);
assert_eq!(
snap.attempts_discarded, 0,
"the last outcome settled the attempt, so there was nothing \
outstanding to discard"
);
}
#[test]
fn a_defaulted_verdict_is_not_futility() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(9);
let p = peer(1);
detector.record_repair_attempt(&key, &p, now);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_outcome(
&key,
&p,
false,
OutcomeEvidence::ProbeBudgetExhausted,
now,
);
}
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::ProbeUnavailable, now);
}
let snap = detector.snapshot();
assert_eq!(
snap.futile, 0,
"a default is not evidence that a repair failed"
);
assert_eq!(snap.would_quarantine, 0);
assert_eq!(
snap.outcomes_probe_budget_exhausted,
u64::from(QUARANTINE_THRESHOLD),
"budget-exhausted defaults get their own row so the headline is \
readable against them"
);
assert_eq!(
snap.outcomes_probe_unavailable,
u64::from(QUARANTINE_THRESHOLD),
"probe failures are a distinct cause from budget exhaustion"
);
assert_eq!(
snap.observations_unpaired, 0,
"an evidence-free comparison is its own class, not an unpaired one"
);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
let snap = detector.snapshot();
assert_eq!(
snap.futile, 1,
"a defaulted comparison must not consume the outstanding attempt"
);
assert_eq!(snap.attempts, 1, "one heal was emitted, so one attempt");
}
#[test]
fn a_defaulted_verdict_does_not_reset_a_streak() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(10);
let p = peer(1);
for _ in 0..(QUARANTINE_THRESHOLD - 1) {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
detector.record_repair_outcome(&key, &p, true, OutcomeEvidence::ProbeUnavailable, now);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
assert_eq!(
detector.snapshot().would_quarantine,
1,
"an evidence-free reading must move the streak in neither direction"
);
}
#[test]
fn repeated_attempts_without_an_outcome_are_counted_as_superseded() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(11);
let p = peer(1);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_attempt(&key, &p, now);
let snap = detector.snapshot();
assert_eq!(snap.attempts, 3);
assert_eq!(snap.attempts_superseded, 2);
assert_eq!(snap.futile, 0);
assert_eq!(
snap.evictions, 0,
"re-recording an attempt on a tracked edge replaces its value; \
that is not an eviction and must never be counted as one — \
`note_eviction` decrements an unsigned gauge"
);
}
#[test]
fn re_recording_an_attempt_on_an_at_threshold_edge_is_not_an_eviction() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(12);
let p = peer(1);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
assert_eq!(detector.snapshot().edges_at_threshold, 1);
detector.record_repair_attempt(&key, &p, now);
let snap = detector.snapshot();
assert_eq!(
snap.evictions, 0,
"replacing a tracked edge's value is not an eviction: {snap:?}"
);
assert_eq!(
snap.evictions_losing_streak, 0,
"nothing was forgotten, so the undercount signal must stay clean: \
{snap:?}"
);
assert_eq!(
snap.edges_at_threshold, 1,
"the at-threshold gauge must survive re-healing a stuck edge: \
{snap:?}"
);
assert_eq!(snap.tracked_edges, 1);
}
#[test]
fn ladder_is_a_monotone_survival_curve() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let p = peer(1);
for (seed, streak) in [(20u8, 1u32), (21, 4), (22, 32)] {
let key = contract(seed);
for _ in 0..streak {
detector.record_repair_attempt(&key, &p, now);
detector.record_repair_outcome(&key, &p, false, OutcomeEvidence::Verdict, now);
}
}
let ladder = detector.snapshot().ladder;
assert_eq!(ladder, [3, 2, 2, 2, 1, 1, 1, 1]);
for window in ladder.windows(2) {
assert!(
window[0] >= window[1],
"survival curve must be non-increasing, got {ladder:?}"
);
}
}
#[test]
fn eviction_is_observable() {
let detector = FutileRepairDetector::with_capacity(2);
let now = t0();
let p = peer(1);
let victim = contract(30);
detector.record_repair_attempt(&victim, &p, now);
detector.record_repair_outcome(&victim, &p, false, OutcomeEvidence::Verdict, now);
for seed in [31u8, 32] {
detector.record_repair_attempt(&contract(seed), &p, now);
}
let snap = detector.snapshot();
assert_eq!(snap.tracked_edges, 2, "occupancy is capped");
assert_eq!(snap.evictions, 1);
assert_eq!(
snap.evictions_losing_streak, 1,
"an evicted streak must be reported, or futility counts silently \
undercount"
);
}
#[test]
fn eviction_releases_the_at_threshold_gauge() {
let detector = FutileRepairDetector::with_capacity(1);
let now = t0();
let p = peer(1);
let victim = contract(40);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&victim, &p, now);
detector.record_repair_outcome(&victim, &p, false, OutcomeEvidence::Verdict, now);
}
assert_eq!(detector.snapshot().edges_at_threshold, 1);
detector.record_repair_attempt(&contract(41), &p, now);
let snap = detector.snapshot();
assert_eq!(snap.edges_at_threshold, 0);
assert_eq!(
snap.would_quarantine, 1,
"the crossing already happened and stays counted"
);
}
#[test]
fn edges_are_per_peer_not_per_contract() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let key = contract(50);
let stuck = peer(1);
let healthy = peer(2);
for _ in 0..QUARANTINE_THRESHOLD {
detector.record_repair_attempt(&key, &stuck, now);
detector.record_repair_outcome(&key, &stuck, false, OutcomeEvidence::Verdict, now);
detector.record_repair_attempt(&key, &healthy, now);
detector.record_repair_outcome(&key, &healthy, true, OutcomeEvidence::Verdict, now);
}
let snap = detector.snapshot();
assert_eq!(snap.would_quarantine, 1);
assert_eq!(snap.edges_at_threshold, 1);
assert_eq!(snap.futile, u64::from(QUARANTINE_THRESHOLD));
assert_eq!(snap.productive, u64::from(QUARANTINE_THRESHOLD));
}
#[test]
fn snapshot_row_order_is_the_documented_wire_contract() {
let snap = FutileRepairSnapshot {
attempts: 1,
futile: 2,
productive: 3,
observations_unpaired: 4,
attempts_superseded: 5,
attempts_discarded: 6,
outcomes_probe_budget_exhausted: 7,
outcomes_probe_unavailable: 8,
outcomes_after_long_gap: 9,
would_quarantine: 10,
edges_at_threshold: 11,
tracked_edges: 12,
evictions: 13,
evictions_losing_streak: 14,
ladder: [0; LADDER_LEN],
};
assert_eq!(
snap.to_row(),
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
);
}
#[test]
fn outcome_rows_partition_every_observation() {
let detector = FutileRepairDetector::with_capacity(16);
let now = t0();
let p = peer(1);
let mut observations = 0u64;
for (seed, converged, evidence) in [
(60u8, false, OutcomeEvidence::Verdict),
(61, true, OutcomeEvidence::Verdict),
(62, false, OutcomeEvidence::ProbeBudgetExhausted),
(63, false, OutcomeEvidence::ProbeUnavailable),
] {
let key = contract(seed);
detector.record_repair_attempt(&key, &p, now);
for _ in 0..3 {
detector.record_repair_outcome(&key, &p, converged, evidence, now);
observations += 1;
}
}
let snap = detector.snapshot();
assert_eq!(
snap.futile
+ snap.productive
+ snap.observations_unpaired
+ snap.outcomes_probe_budget_exhausted
+ snap.outcomes_probe_unavailable,
observations,
"every call to record_repair_outcome must land in exactly one of \
the five outcome rows: {snap:?}"
);
}
}