use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use super::{BenchConfig, BenchResults, Shared};
use crate::broker::router::RoutingStrategy;
pub(crate) const STRATEGIES: [RoutingStrategy; 6] = [
RoutingStrategy::BestLatency,
RoutingStrategy::BestPrice,
RoutingStrategy::RoundRobin,
RoutingStrategy::WeightedCapacity,
RoutingStrategy::Random,
RoutingStrategy::BestAvailability,
];
pub(crate) const DEFAULT_REQUESTS_EACH: u64 = 300;
#[derive(Default)]
pub(crate) struct Progress {
current: Mutex<Option<(RoutingStrategy, Shared, Instant)>>,
done: Mutex<Vec<(RoutingStrategy, BenchResults)>>,
started: AtomicUsize,
stop: AtomicBool,
over: AtomicBool,
}
impl Progress {
pub(crate) fn new() -> Arc<Progress> {
Arc::new(Progress::default())
}
fn begin(&self, strategy: RoutingStrategy) {
self.started.fetch_add(1, Ordering::Relaxed);
if let Ok(mut c) = self.current.lock() {
*c = None;
}
let _ = strategy;
}
fn publish(&self, strategy: RoutingStrategy, shared: &Shared) {
if let Ok(mut c) = self.current.lock() {
*c = Some((strategy, shared.clone(), Instant::now()));
}
}
fn finish(&self, strategy: RoutingStrategy, results: BenchResults) {
if let Ok(mut d) = self.done.lock() {
d.push((strategy, results));
}
if let Ok(mut c) = self.current.lock() {
*c = None;
}
}
pub(crate) fn running(&self) -> Option<(RoutingStrategy, Shared, Instant)> {
self.current
.lock()
.ok()
.and_then(|c| c.as_ref().map(|(s, sh, t)| (*s, sh.clone(), *t)))
}
pub(crate) fn finished(&self) -> Vec<(RoutingStrategy, BenchResults)> {
self.done.lock().map(|d| d.clone()).unwrap_or_default()
}
pub(crate) fn started(&self) -> usize {
self.started.load(Ordering::Relaxed)
}
pub(crate) fn is_over(&self) -> bool {
self.over.load(Ordering::Relaxed)
}
pub(crate) fn asked_to_stop(&self) -> bool {
self.stop.load(Ordering::Relaxed)
}
pub(crate) fn ask_to_stop(&self) {
self.stop.store(true, Ordering::Relaxed);
}
pub(crate) fn abandon(&self) {
self.stop.store(true, Ordering::Relaxed);
if let Ok(c) = self.current.lock() {
if let Some((_, shared, _)) = c.as_ref() {
shared.stop.store(true, Ordering::Relaxed);
}
}
}
}
pub(crate) fn run_all(
base: BenchConfig,
progress: Arc<Progress>,
) -> Vec<(RoutingStrategy, BenchResults)> {
for strategy in STRATEGIES {
if progress.asked_to_stop() || super::interrupted() {
break;
}
let mut config = base.clone();
config.strategy = strategy;
config.silent = true;
config.no_progress = true;
progress.begin(strategy);
let p = progress.clone();
let results = super::run_benchmark_watched(config, &|shared| p.publish(strategy, shared));
progress.finish(strategy, results);
}
progress.over.store(true, Ordering::Relaxed);
progress.finished()
}
pub(crate) const MEANINGFUL_MARGIN: f64 = 0.05;
pub(crate) struct Verdicts {
pub(crate) fastest: Option<usize>,
pub(crate) cheapest: Option<usize>,
pub(crate) freest: Option<usize>,
pub(crate) rps_spread: f64,
}
pub(crate) fn free_share(r: &BenchResults) -> f64 {
match r.ledger.placed() {
0 => 0.0,
placed => r.ledger.totals(super::metrics::Placement::Free).0 as f64 / placed as f64,
}
}
fn usable(r: &BenchResults) -> bool {
r.successful > 0
}
fn clear_leader(
results: &[(RoutingStrategy, BenchResults)],
key: impl Fn(&BenchResults) -> f64,
better: impl Fn(f64, f64) -> std::cmp::Ordering,
relative_gap: impl Fn(f64, f64) -> f64,
) -> Option<usize> {
let mut ranked: Vec<(usize, f64)> = results
.iter()
.enumerate()
.filter(|(_, (_, r))| usable(r))
.map(|(i, (_, r))| (i, key(r)))
.collect();
ranked.sort_by(|a, b| better(a.1, b.1));
match ranked.len() {
0 => None,
1 => Some(ranked[0].0),
_ => match relative_gap(ranked[0].1, ranked[1].1) > MEANINGFUL_MARGIN {
true => Some(ranked[0].0),
false => None,
},
}
}
pub(crate) fn verdicts(results: &[(RoutingStrategy, BenchResults)]) -> Verdicts {
let rates: Vec<f64> = results
.iter()
.filter(|(_, r)| usable(r))
.map(|(_, r)| r.rps)
.collect();
let rps_spread = match rates.len() {
0 | 1 => 0.0,
_ => {
let hi = rates.iter().copied().fold(f64::NEG_INFINITY, f64::max);
let lo = rates.iter().copied().fold(f64::INFINITY, f64::min);
match hi > 0.0 {
true => (hi - lo) / hi,
false => 0.0,
}
}
};
Verdicts {
fastest: clear_leader(
results,
|r| r.rps,
|a, b| b.total_cmp(&a),
|best, second| match best > 0.0 {
true => (best - second) / best,
false => 0.0,
},
),
cheapest: clear_leader(
results,
|r| r.total_charged_zkcr,
|a, b| a.total_cmp(&b),
|best, second| match second > 0.0 {
true => (second - best) / second,
false => 0.0,
},
),
freest: match spread(results) > 0.01 {
true => clear_leader(
results,
free_share,
|a, b| b.total_cmp(&a),
|best, second| best - second,
),
false => None,
},
rps_spread,
}
}
fn spread(results: &[(RoutingStrategy, BenchResults)]) -> f64 {
let shares: Vec<f64> = results
.iter()
.filter(|(_, r)| usable(r))
.map(|(_, r)| free_share(r))
.collect();
match shares.len() {
0 | 1 => 0.0,
_ => {
let lo = shares.iter().copied().fold(f64::INFINITY, f64::min);
let hi = shares.iter().copied().fold(f64::NEG_INFINITY, f64::max);
hi - lo
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn run(rps: f64, zkcr: f64, ok: u64) -> BenchResults {
let mut r = BenchResults::new();
r.rps = rps;
r.total_charged_zkcr = zkcr;
r.successful = ok;
r
}
#[test]
fn a_comparison_costs_a_fraction_of_six_full_runs() {
let single = BenchConfig::default().requests;
assert!(
DEFAULT_REQUESTS_EACH < single,
"{DEFAULT_REQUESTS_EACH} per strategy is not smaller than the single-run {single}"
);
assert!(
DEFAULT_REQUESTS_EACH * STRATEGIES.len() as u64 <= single * 2,
"the whole comparison should cost about two single runs"
);
}
#[test]
fn an_explicit_n_is_honoured() {
let c = BenchConfig {
requests: 10_000,
requests_explicit: true,
..Default::default()
};
assert!(c.requests_explicit && c.requests == 10_000);
}
#[test]
fn the_two_strategies_people_choose_between_are_measured_first() {
assert_eq!(STRATEGIES[0], RoutingStrategy::BestLatency);
assert_eq!(STRATEGIES[1], RoutingStrategy::BestPrice);
}
#[test]
fn the_verdicts_name_the_fastest_and_cheapest_when_they_lead_clearly() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.9, 300)),
(RoutingStrategy::BestPrice, run(300.0, 0.1, 300)),
];
let v = verdicts(&results);
assert_eq!(v.fastest, Some(0), "400 vs 300 is a 25% lead");
assert_eq!(v.cheapest, Some(1), "0.1 vs 0.9 is 89% cheaper");
}
#[test]
fn a_one_percent_lead_names_no_winner() {
let results = vec![
(RoutingStrategy::BestLatency, run(2841.9, 0.0, 300)),
(RoutingStrategy::BestPrice, run(2847.8, 0.0, 300)),
(RoutingStrategy::RoundRobin, run(2848.0, 0.0, 300)),
(RoutingStrategy::BestAvailability, run(2889.9, 0.0, 300)),
];
let v = verdicts(&results);
assert!(v.fastest.is_none(), "1.5% is noise, not a winner");
assert!(v.cheapest.is_none(), "all of them spent nothing");
assert!(
(v.rps_spread - 0.0166).abs() < 0.001,
"the spread is the finding: {}",
v.rps_spread
);
}
#[test]
fn everything_spending_zero_names_no_cheapest() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.0, 300)),
(RoutingStrategy::BestPrice, run(100.0, 0.0, 300)),
];
let v = verdicts(&results);
assert!(v.cheapest.is_none(), "nothing separates them on cost");
assert_eq!(v.fastest, Some(0), "but speed still separates them");
}
#[test]
fn free_beats_paid_even_though_the_gap_is_measured_against_zero() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.0, 300)),
(RoutingStrategy::BestPrice, run(400.0, 2.5, 300)),
];
assert_eq!(verdicts(&results).cheapest, Some(0));
}
#[test]
fn the_only_strategy_that_measured_anything_is_the_answer() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.9, 300)),
(RoutingStrategy::Random, run(0.0, 0.0, 0)),
];
let v = verdicts(&results);
assert_eq!(v.fastest, Some(0));
assert_eq!(v.cheapest, Some(0));
}
#[test]
fn most_free_is_not_reported_when_every_strategy_is_equally_free() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.0, 300)),
(RoutingStrategy::Random, run(200.0, 0.0, 300)),
];
assert!(verdicts(&results).freest.is_none());
}
#[test]
fn most_free_is_reported_when_they_differ() {
let mut mixed = run(200.0, 0.4, 300);
mixed
.ledger
.record("i9-w0", Some("quic"), Some(3.6), 1000, 0.4, None);
let mut all_free = run(400.0, 0.0, 300);
all_free
.ledger
.record("a-w0", Some("local"), Some(0.0), 1000, 0.0, None);
let results = vec![
(RoutingStrategy::BestLatency, mixed),
(RoutingStrategy::BestPrice, all_free),
];
assert_eq!(verdicts(&results).freest, Some(1));
}
#[test]
fn a_strategy_that_measured_nothing_wins_nothing() {
let results = vec![
(RoutingStrategy::BestLatency, run(400.0, 0.9, 300)),
(RoutingStrategy::BestPrice, run(390.0, 0.8, 300)),
(RoutingStrategy::Random, run(0.0, 0.0, 0)),
];
assert_ne!(verdicts(&results).cheapest, Some(2));
}
#[test]
fn nothing_measured_at_all_names_no_winner() {
let v = verdicts(&[(RoutingStrategy::Random, run(0.0, 0.0, 0))]);
assert!(v.fastest.is_none() && v.cheapest.is_none() && v.freest.is_none());
}
#[test]
fn stopping_lets_the_strategy_in_flight_finish() {
let p = Progress::new();
assert!(!p.asked_to_stop());
p.ask_to_stop();
assert!(p.asked_to_stop());
assert!(p.running().is_none());
}
}