use serde::{Deserialize, Serialize};
use crate::calibration::brier_score;
use crate::comparison_sets::{comparison_set, restrict_to_shared, TaggedRun, TaggedSubmission};
use crate::decay::edge_half_life;
use crate::deflated_sharpe::{deflated_sharpe_ratio, probabilistic_sharpe_ratio, sharpe_ratio};
use crate::econrationality::{elicit_revealed_selection, rationality_score};
use crate::pass_k::{pass_k, PassMode};
use crate::percentile::percentile_of;
use crate::process::{process_score, ProcessEvent, ProcessScore, Trace};
use crate::rediscovery::{clone_clusters, CLONE_COLLAPSE_COSINE};
use crate::roles::{attribute_behavior_roles, RoleContribution};
use crate::rolling::rolling_sharpe;
use crate::selection::{selection_robustness, SelectionRobustness};
use crate::significance::bootstrap_pvalue;
use crate::stats::{mean, std_dev};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Run {
pub returns: Vec<f64>,
#[serde(default)]
pub trace: Trace,
#[serde(default)]
pub confidences: Vec<f64>,
#[serde(default)]
pub outcomes: Vec<bool>,
#[serde(default)]
pub cost: f64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct AgentSubmission {
pub agent_id: String,
pub runs: Vec<Run>,
#[serde(default)]
pub in_sample_trials: u32,
#[serde(default)]
pub candidates: Vec<Vec<f64>>,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub enum RankKey {
#[default]
DeflatedSharpe,
Alpha,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Mandate {
pub max_drawdown: f64,
#[serde(default = "default_max_run_drawdown")]
pub max_run_drawdown: f64,
}
impl Default for Mandate {
fn default() -> Self {
Self {
max_drawdown: 1.0,
max_run_drawdown: default_max_run_drawdown(),
}
}
}
fn default_max_run_drawdown() -> f64 {
1.0
}
fn max_drawdown(returns: &[f64]) -> f64 {
let mut nav = 1.0;
let mut peak = 1.0;
let mut mdd = 0.0;
for &r in returns {
nav *= 1.0 + r;
if nav > peak {
peak = nav;
}
if peak > 0.0 {
let dd = 1.0 - nav / peak;
if dd > mdd {
mdd = dd;
}
}
}
mdd
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreConfig {
pub n_trials: u32,
pub trials_sr_std: f64,
pub dsr_bar: f64,
pub per_run_psr_bar: f64,
#[serde(default)]
pub per_run_min_annual_sharpe: f64,
#[serde(default)]
pub pass_mode: PassMode,
#[serde(default = "default_benchmark_agent_id")]
pub benchmark_agent_id: String,
#[serde(default = "default_periods_per_year")]
pub periods_per_year: f64,
pub alpha: f64,
pub bootstrap_seed: u64,
pub n_boot: usize,
pub block_prob: f64,
#[serde(default)]
pub mandate: Mandate,
#[serde(default)]
pub rank_key: RankKey,
#[serde(default)]
pub reference_dsr_population: Vec<f64>,
#[serde(default = "default_rolling_window")]
pub rolling_window: usize,
#[serde(default = "default_dsr_ci_level")]
pub dsr_ci_level: f64,
#[serde(default = "default_shared_run_set")]
pub shared_run_set: bool,
#[serde(default = "default_min_field_for_measured_sr_std")]
pub min_field_for_measured_sr_std: usize,
#[serde(default = "default_dedup_clones_for_measured_sr_std")]
pub dedup_clones_for_measured_sr_std: bool,
}
fn default_rolling_window() -> usize {
21
}
fn default_dsr_ci_level() -> f64 {
0.90
}
fn default_shared_run_set() -> bool {
true
}
fn default_min_field_for_measured_sr_std() -> usize {
5
}
fn default_dedup_clones_for_measured_sr_std() -> bool {
true
}
fn default_periods_per_year() -> f64 {
252.0
}
fn default_benchmark_agent_id() -> String {
"buy-and-hold".to_string()
}
pub fn per_run_passes(
sub: &AgentSubmission,
benchmark: Option<&AgentSubmission>,
cfg: &ScoreConfig,
) -> Vec<bool> {
let bar = per_run_psr_benchmark(cfg);
match cfg.pass_mode {
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, bar) >= cfg.per_run_psr_bar)
.collect(),
PassMode::RelativeToBenchmark => sub
.runs
.iter()
.enumerate()
.map(|(i, r)| {
benchmark
.and_then(|b| b.runs.get(i))
.and_then(|b| excess_returns(&r.returns, &b.returns))
.is_some_and(|e| {
std_dev(&e) > 0.0
&& probabilistic_sharpe_ratio(&e, bar) >= cfg.per_run_psr_bar
})
})
.collect(),
}
}
fn excess_returns(returns: &[f64], benchmark: &[f64]) -> Option<Vec<f64>> {
(returns.len() == benchmark.len())
.then(|| returns.iter().zip(benchmark).map(|(a, b)| a - b).collect())
}
pub fn per_period_sr_std(cfg: &ScoreConfig) -> f64 {
cfg.trials_sr_std / cfg.periods_per_year.sqrt()
}
pub fn per_run_psr_benchmark(cfg: &ScoreConfig) -> f64 {
cfg.per_run_min_annual_sharpe / cfg.periods_per_year.sqrt()
}
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TrialsSrStdSource {
#[default]
Configured,
Measured,
}
impl Default for ScoreConfig {
fn default() -> Self {
Self {
n_trials: 50,
trials_sr_std: 0.5,
dsr_bar: 0.95,
per_run_psr_bar: 0.90,
per_run_min_annual_sharpe: 0.0,
pass_mode: PassMode::default(),
benchmark_agent_id: default_benchmark_agent_id(),
periods_per_year: default_periods_per_year(),
alpha: 0.05,
bootstrap_seed: 0x5BA7_2026,
n_boot: 2000,
block_prob: 0.1,
mandate: Mandate::default(),
rank_key: RankKey::default(),
reference_dsr_population: Vec::new(),
rolling_window: default_rolling_window(),
dsr_ci_level: default_dsr_ci_level(),
shared_run_set: default_shared_run_set(),
min_field_for_measured_sr_std: default_min_field_for_measured_sr_std(),
dedup_clones_for_measured_sr_std: default_dedup_clones_for_measured_sr_std(),
}
}
}
impl ScoreConfig {
pub fn for_periods_per_year(periods_per_year: f64) -> Self {
Self {
periods_per_year,
..Self::default()
}
}
pub fn reliability_never_catastrophic(max_run_dd: f64) -> Self {
Self {
pass_mode: PassMode::Any,
mandate: Mandate {
max_run_drawdown: max_run_dd,
..Mandate::default()
},
..Self::default()
}
}
pub fn relative_to_benchmark(benchmark_agent_id: &str) -> Self {
Self {
pass_mode: PassMode::RelativeToBenchmark,
benchmark_agent_id: benchmark_agent_id.to_string(),
..Self::default()
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct CompositeScore {
pub agent_id: String,
pub deflated_sharpe: f64,
pub psr: f64,
pub passed_k: bool,
pub process_ok: bool,
pub bootstrap_p: f64,
pub raw_mean_return: f64,
pub rank_eligible: bool,
pub composite: f64,
pub alpha: f64,
pub beta: f64,
pub calibration_brier: Option<f64>,
pub edge_half_life: Option<f64>,
pub field_reality_check_p: f64,
pub max_drawdown: f64,
pub mandate_ok: bool,
#[serde(default)]
pub worst_run_drawdown: f64,
pub turnover: f64,
pub pareto_optimal: bool,
pub step_down_significant: bool,
pub confidence_weighted_return: f64,
pub cost: f64,
pub return_per_cost: Option<f64>,
pub field_spa_p: f64,
pub field_spa_consistent_p: f64,
pub field_crowdedness: Option<f64>,
pub in_sample_trials: u32,
pub effective_n_trials: u32,
pub dsr_percentile: Option<f64>,
pub selection_median_dsr: Option<f64>,
pub selection_gap: Option<f64>,
pub rank_ordinal: usize,
pub rolling_min_sharpe: Option<f64>,
pub rolling_frac_positive: Option<f64>,
pub sortino: Option<f64>,
pub downside_deviation: f64,
pub dsr_per_cost: Option<f64>,
pub process_floored: bool,
pub realized_floored_return: f64,
pub dsr_ci_low: f64,
pub dsr_ci_high: f64,
pub dsr_se: f64,
pub tie_group: usize,
pub dsr_tied: bool,
#[serde(default)]
pub trials_sr_std: f64,
#[serde(default)]
pub trials_sr_std_annualized: Option<f64>,
#[serde(default)]
pub trials_sr_std_source: TrialsSrStdSource,
#[serde(default)]
pub runs_submitted: usize,
#[serde(default)]
pub runs_scored: usize,
#[serde(default = "default_process_score")]
pub process_score: f64,
#[serde(default)]
pub process_warnings: usize,
#[serde(default)]
pub econ_rationality_score: Option<f64>,
#[serde(default)]
pub econ_dominance_violations: Option<usize>,
#[serde(default)]
pub role_contributions: Vec<RoleContribution>,
}
fn default_process_score() -> f64 {
1.0
}
fn dominates(a: &CompositeScore, b: &CompositeScore) -> bool {
a.raw_mean_return >= b.raw_mean_return
&& a.max_drawdown <= b.max_drawdown
&& a.turnover <= b.turnover
&& (a.raw_mean_return > b.raw_mean_return
|| a.max_drawdown < b.max_drawdown
|| a.turnover < b.turnover)
}
#[derive(Clone, Copy)]
struct Deflation {
sr_std: f64,
annualized: Option<f64>,
source: TrialsSrStdSource,
}
impl Deflation {
fn configured(cfg: &ScoreConfig) -> Self {
Self {
sr_std: per_period_sr_std(cfg),
annualized: Some(cfg.trials_sr_std),
source: TrialsSrStdSource::Configured,
}
}
fn measured(sr_std: f64) -> Self {
Self {
sr_std,
annualized: None,
source: TrialsSrStdSource::Measured,
}
}
}
pub fn score_agent(sub: &AgentSubmission, cfg: &ScoreConfig) -> CompositeScore {
score_agent_with(sub, cfg, Deflation::configured(cfg), None)
}
fn score_agent_with(
sub: &AgentSubmission,
cfg: &ScoreConfig,
defl: Deflation,
benchmark: Option<&AgentSubmission>,
) -> CompositeScore {
let pooled: Vec<f64> = sub
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
let psr = probabilistic_sharpe_ratio(&pooled, 0.0);
let effective_n_trials = cfg.n_trials.saturating_add(sub.in_sample_trials);
let dsr = deflated_sharpe_ratio(&pooled, effective_n_trials, defl.sr_std);
let per_run = per_run_passes(sub, benchmark, cfg);
let passed_k = pass_k(&per_run, cfg.pass_mode);
let per_run_process: Vec<ProcessScore> =
sub.runs.iter().map(|r| process_score(&r.trace)).collect();
let process_ok = per_run_process.iter().all(ProcessScore::is_clean);
let process_warnings: usize = per_run_process.iter().map(|p| p.warn_violations).sum();
let graded_process_score = if process_ok {
(1.0 - process_warnings as f64 * 0.1).max(0.0)
} else {
0.0
};
let bootstrap_p = bootstrap_pvalue(&pooled, cfg.bootstrap_seed, cfg.n_boot, cfg.block_prob);
let raw_mean_return = mean(&pooled);
let conf: Vec<f64> = sub
.runs
.iter()
.flat_map(|r| r.confidences.iter().copied())
.collect();
let outc: Vec<bool> = sub
.runs
.iter()
.flat_map(|r| r.outcomes.iter().copied())
.collect();
let calibration_brier = if !conf.is_empty() && !outc.is_empty() {
Some(brier_score(&conf, &outc))
} else {
None
};
let per_run_edge: Vec<f64> = sub.runs.iter().map(|r| mean(&r.returns)).collect();
let edge_half_life_periods = edge_half_life(&per_run_edge);
let mdd = max_drawdown(&pooled);
let worst_run_drawdown = sub
.runs
.iter()
.map(|r| max_drawdown(&r.returns))
.fold(0.0, f64::max);
let mandate_ok =
mdd <= cfg.mandate.max_drawdown && worst_run_drawdown <= cfg.mandate.max_run_drawdown;
let total_orders: usize = sub
.runs
.iter()
.map(|r| {
r.trace
.events
.iter()
.filter(|e| matches!(e, ProcessEvent::OrderPlaced { .. }))
.count()
})
.sum();
let turnover = total_orders as f64 / sub.runs.len().max(1) as f64;
let mut cw_num = 0.0;
let mut cw_den = 0.0;
for r in &sub.runs {
let w = if r.confidences.is_empty() {
1.0
} else {
mean(&r.confidences)
};
cw_num += w * mean(&r.returns);
cw_den += w;
}
let confidence_weighted_return = if cw_den > 0.0 {
cw_num / cw_den
} else {
raw_mean_return
};
let cost: f64 = sub.runs.iter().map(|r| r.cost).sum();
let return_per_cost = if cost > 0.0 {
Some(raw_mean_return / cost)
} else {
None
};
let dsr_percentile = if cfg.reference_dsr_population.is_empty() {
None
} else {
Some(percentile_of(dsr, &cfg.reference_dsr_population))
};
let (selection_median_dsr, selection_gap) = if sub.candidates.is_empty() {
(None, None)
} else {
let sr: SelectionRobustness =
selection_robustness(&sub.candidates, effective_n_trials, defl.sr_std);
(Some(sr.median_dsr), Some(sr.selection_gap))
};
let rolling = rolling_sharpe(&pooled, cfg.rolling_window);
let rolling_min_sharpe = rolling.map(|r| r.min_sharpe);
let rolling_frac_positive = rolling.map(|r| r.frac_positive);
let sortino = crate::stats::sortino_ratio(&pooled, 0.0);
let downside_deviation = crate::stats::downside_deviation(&pooled, 0.0);
let dsr_per_cost = if cost > 0.0 { Some(dsr / cost) } else { None };
let process_floored = !process_ok;
let realized_floored_return = if process_floored {
0.0
} else {
raw_mean_return
};
let dsr_ci = crate::significance::bootstrap_dsr_ci(
&pooled,
effective_n_trials,
defl.sr_std,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.dsr_ci_level,
);
let econ_choice = elicit_revealed_selection(&sub.candidates, &pooled);
let (econ_rationality_score, econ_dominance_violations) = match econ_choice {
Some(choice) => {
let violations = usize::from(choice.is_dominated());
(Some(rationality_score(&[choice])), Some(violations))
}
None => (None, None),
};
let role_contributions = attribute_behavior_roles(&sub.runs);
let rank_eligible =
dsr >= cfg.dsr_bar && passed_k && process_ok && bootstrap_p < cfg.alpha && mandate_ok;
let composite = if rank_eligible { dsr } else { 0.0 };
CompositeScore {
agent_id: sub.agent_id.clone(),
deflated_sharpe: dsr,
psr,
passed_k,
process_ok,
bootstrap_p,
raw_mean_return,
rank_eligible,
composite,
alpha: 0.0,
beta: 0.0,
calibration_brier,
edge_half_life: edge_half_life_periods,
field_reality_check_p: 1.0,
max_drawdown: mdd,
mandate_ok,
worst_run_drawdown,
turnover,
pareto_optimal: false,
step_down_significant: false,
confidence_weighted_return,
cost,
return_per_cost,
field_spa_p: 1.0,
field_spa_consistent_p: 1.0,
field_crowdedness: None,
in_sample_trials: sub.in_sample_trials,
effective_n_trials,
dsr_percentile,
selection_median_dsr,
selection_gap,
rank_ordinal: 0,
rolling_min_sharpe,
rolling_frac_positive,
sortino,
downside_deviation,
dsr_per_cost,
process_floored,
realized_floored_return,
dsr_ci_low: dsr_ci.lower,
dsr_ci_high: dsr_ci.upper,
dsr_se: dsr_ci.se,
tie_group: 0,
dsr_tied: false,
trials_sr_std: defl.sr_std,
trials_sr_std_annualized: defl.annualized,
trials_sr_std_source: defl.source,
runs_submitted: sub.runs.len(),
runs_scored: sub.runs.len(),
process_score: graded_process_score,
process_warnings,
econ_rationality_score,
econ_dominance_violations,
role_contributions,
}
}
fn restrict_to_shared_positions(subs: &[AgentSubmission]) -> Vec<AgentSubmission> {
let tag = |i: usize| format!("{i:08}");
let tagged: Vec<TaggedSubmission> = subs
.iter()
.filter(|s| !s.runs.is_empty())
.map(|s| TaggedSubmission {
agent_id: s.agent_id.clone(),
runs: s
.runs
.iter()
.enumerate()
.map(|(i, run)| TaggedRun {
window_id: tag(i),
run: run.clone(),
})
.collect(),
in_sample_trials: s.in_sample_trials,
candidates: s.candidates.clone(),
})
.collect();
let roster: Vec<String> = tagged.iter().map(|s| s.agent_id.clone()).collect();
let set = comparison_set(&roster, &tagged);
let mut tagged_iter = tagged.iter();
subs.iter()
.map(|s| {
if s.runs.is_empty() {
return s.clone();
}
let t = tagged_iter
.next()
.expect("one tagged submission per non-empty submission");
restrict_to_shared(&set, t)
})
.collect()
}
fn measured_trials_sr_std(
pooled: &[Vec<f64>],
min_field: usize,
dedup_clones: bool,
) -> Option<f64> {
let qualifying: Vec<(&[f64], f64)> = pooled
.iter()
.filter(|p| p.len() >= 2)
.map(|p| (p.as_slice(), sharpe_ratio(p)))
.filter(|(_, sr)| sr.is_finite())
.collect();
let mut sharpes: Vec<f64> = if dedup_clones {
let streams: Vec<Vec<f64>> = qualifying.iter().map(|(p, _)| p.to_vec()).collect();
clone_clusters(&streams, CLONE_COLLAPSE_COSINE, false)
.iter()
.map(|members| {
let mut cluster: Vec<f64> = members.iter().map(|&i| qualifying[i].1).collect();
cluster.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
cluster[(cluster.len() - 1) / 2]
})
.collect()
} else {
qualifying.iter().map(|(_, sr)| *sr).collect()
};
if sharpes.len() < min_field.max(2) {
return None;
}
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
Some(std_dev(&sharpes))
}
pub fn rank(subs: &[AgentSubmission], cfg: &ScoreConfig) -> Vec<CompositeScore> {
let restricted;
let field: &[AgentSubmission] = if cfg.shared_run_set {
restricted = restrict_to_shared_positions(subs);
&restricted
} else {
subs
};
let pooled: Vec<Vec<f64>> = field
.iter()
.map(|s| {
s.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
})
.collect();
let min_len = pooled.iter().map(Vec::len).min().unwrap_or(0);
let n_agents = pooled.len().max(1) as f64;
let market: Vec<f64> = (0..min_len)
.map(|i| pooled.iter().map(|p| p[i]).sum::<f64>() / n_agents)
.collect();
let defl = measured_trials_sr_std(
&pooled,
cfg.min_field_for_measured_sr_std,
cfg.dedup_clones_for_measured_sr_std,
)
.map_or_else(|| Deflation::configured(cfg), Deflation::measured);
let benchmark = match cfg.pass_mode {
PassMode::RelativeToBenchmark => {
field.iter().find(|s| s.agent_id == cfg.benchmark_agent_id)
}
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => None,
};
let mut scores: Vec<CompositeScore> = field
.iter()
.enumerate()
.map(|(idx, s)| {
let mut cs = score_agent_with(s, cfg, defl, benchmark);
cs.runs_submitted = subs[idx].runs.len();
if min_len >= 2 {
let (alpha, beta) = crate::attribution::alpha_beta(&pooled[idx], &market);
cs.alpha = alpha;
cs.beta = beta;
}
cs
})
.collect();
if min_len >= 2 {
let field_excess: Vec<Vec<f64>> = pooled
.iter()
.map(|p| {
p.iter()
.take(min_len)
.zip(market.iter())
.map(|(a, m)| a - m)
.collect()
})
.collect();
let rc_p = crate::significance::reality_check_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_p = crate::significance::spa_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_c_p = crate::significance::spa_consistent_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
for cs in scores.iter_mut() {
cs.field_reality_check_p = rc_p;
cs.field_spa_p = spa_p;
cs.field_spa_consistent_p = spa_c_p;
}
let sd = crate::significance::step_down_significant(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.alpha,
);
for (cs, s) in scores.iter_mut().zip(sd) {
cs.step_down_significant = s;
}
}
if min_len >= 2 && pooled.len() >= 2 {
let aligned: Vec<&[f64]> = pooled.iter().map(|p| &p[..min_len]).collect();
for (idx, cs) in scores.iter_mut().enumerate() {
let peers: Vec<&[f64]> = aligned
.iter()
.enumerate()
.filter(|&(j, _)| j != idx)
.map(|(_, &p)| p)
.collect();
cs.field_crowdedness = crate::correlation::crowdedness(aligned[idx], &peers).mean_corr;
}
}
let pareto: Vec<bool> = (0..scores.len())
.map(|i| !(0..scores.len()).any(|j| j != i && dominates(&scores[j], &scores[i])))
.collect();
for (cs, p) in scores.iter_mut().zip(pareto) {
cs.pareto_optimal = p;
}
let sort_key = |s: &CompositeScore| match cfg.rank_key {
RankKey::DeflatedSharpe => s.composite,
RankKey::Alpha => {
if s.rank_eligible {
s.alpha
} else {
f64::NEG_INFINITY
}
}
};
scores.sort_by(|a, b| {
b.rank_eligible
.cmp(&a.rank_eligible)
.then(
sort_key(b)
.partial_cmp(&sort_key(a))
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
b.raw_mean_return
.partial_cmp(&a.raw_mean_return)
.unwrap_or(std::cmp::Ordering::Equal),
)
});
let mut group = 0usize;
let mut prev: Option<usize> = None;
for i in 0..scores.len() {
if !scores[i].rank_eligible {
continue;
}
let same_band = matches!(prev, Some(p) if ci_overlap(&scores[p], &scores[i]));
if !same_band {
group += 1;
}
scores[i].tie_group = group;
prev = Some(i);
}
let mut i = 0;
while i < scores.len() {
if !scores[i].rank_eligible {
i += 1;
continue;
}
let band = scores[i].tie_group;
let mut j = i + 1;
while j < scores.len() && scores[j].rank_eligible && scores[j].tie_group == band {
j += 1;
}
scores[i..j].sort_by(|a, b| {
b.process_score
.partial_cmp(&a.process_score)
.unwrap_or(std::cmp::Ordering::Equal)
});
i = j;
}
let mut ord = 0usize;
for cs in scores.iter_mut() {
if cs.rank_eligible {
ord += 1;
cs.rank_ordinal = ord;
}
}
let mut band_counts = vec![0usize; group + 1];
for cs in &scores {
if cs.rank_eligible {
band_counts[cs.tie_group] += 1;
}
}
for cs in scores.iter_mut() {
if cs.rank_eligible {
cs.dsr_tied = band_counts[cs.tie_group] > 1;
}
}
scores
}
fn ci_overlap(a: &CompositeScore, b: &CompositeScore) -> bool {
a.dsr_ci_low <= b.dsr_ci_high && b.dsr_ci_low <= a.dsr_ci_high
}
#[cfg(test)]
mod tests {
use super::*;
use crate::deflated_sharpe::expected_max_sharpe;
use crate::process::ProcessEvent;
fn run(mean_ret: f64, amp: f64, n: usize) -> Run {
let returns = (0..n)
.map(|i| mean_ret + amp * (i as f64 * 0.7).sin())
.collect();
Run {
returns,
trace: Trace::default(),
confidences: Vec::new(),
outcomes: Vec::new(),
cost: 0.0,
}
}
fn agent(id: &str, runs: Vec<Run>) -> AgentSubmission {
AgentSubmission {
agent_id: id.to_string(),
runs,
in_sample_trials: 0,
candidates: Vec::new(),
}
}
#[test]
fn skilled_is_eligible() {
let s = score_agent(
&agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rank_eligible, "skilled should be eligible: {s:?}");
assert!(s.passed_k && s.process_ok);
}
#[test]
fn lucky_high_return_fails_pass_k() {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
let s = score_agent(&agent("lucky", runs), &ScoreConfig::default());
assert!(!s.passed_k, "lucky should fail pass^k");
assert!(!s.rank_eligible, "lucky must not be rank-eligible: {s:?}");
}
#[test]
fn process_violator_is_disqualified() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("violator", runs), &ScoreConfig::default());
assert!(!s.process_ok);
assert!(!s.rank_eligible, "a risk-gate bypass must disqualify");
}
#[test]
fn deflation_demotes_luck() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky.clone(), skilled.clone()], &ScoreConfig::default());
let lucky_raw = board
.iter()
.find(|s| s.agent_id == "lucky")
.unwrap()
.raw_mean_return;
let skilled_raw = board
.iter()
.find(|s| s.agent_id == "skilled")
.unwrap()
.raw_mean_return;
assert!(
lucky_raw > skilled_raw,
"lucky raw {lucky_raw} should exceed skilled {skilled_raw}"
);
assert_eq!(board[0].agent_id, "skilled");
assert!(board[0].rank_eligible && !board[1].rank_eligible);
}
#[test]
fn confidence_weighting_rewards_conviction() {
let win = Run {
returns: vec![0.01; 30],
trace: Trace::default(),
confidences: vec![0.9; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let lose = Run {
returns: vec![-0.005; 30],
trace: Trace::default(),
confidences: vec![0.1; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let s = score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default());
assert!(
s.confidence_weighted_return > s.raw_mean_return,
"cwr {} should beat raw {}",
s.confidence_weighted_return,
s.raw_mean_return
);
}
#[test]
fn cost_efficiency_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 30);
r.cost = 4.0;
let s = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
assert_eq!(s.cost, 4.0);
assert!(s.return_per_cost.is_some());
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 30)]),
&ScoreConfig::default(),
);
assert!(free.return_per_cost.is_none());
}
#[test]
fn in_sample_search_raises_the_deflation_bar() {
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let base = score_agent(&agent("base", runs.clone()), &ScoreConfig::default());
let mut over = agent("over", runs);
over.in_sample_trials = 5000;
let s = score_agent(&over, &ScoreConfig::default());
assert_eq!(s.effective_n_trials, 5050);
assert!(
s.deflated_sharpe <= base.deflated_sharpe,
"more in-sample search must not raise DSR ({} vs {})",
s.deflated_sharpe,
base.deflated_sharpe
);
}
#[test]
fn percentile_reported_only_with_reference() {
let none = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(none.dsr_percentile.is_none());
let cfg = ScoreConfig {
reference_dsr_population: vec![0.0, 0.3, 0.6, 0.9],
..ScoreConfig::default()
};
let some = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&cfg,
);
assert!(some.dsr_percentile.is_some());
}
#[test]
fn rolling_sharpe_reported_for_long_tracks() {
let s = score_agent(
&agent("roll", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rolling_min_sharpe.is_some());
let fp = s.rolling_frac_positive.expect("reported");
assert!(
(fp - 1.0).abs() < 1e-12,
"steady edge → all windows positive"
);
}
#[test]
fn rolling_sharpe_none_when_track_too_short() {
let cfg = ScoreConfig {
rolling_window: 100,
..ScoreConfig::default()
};
let s = score_agent(&agent("short", vec![run(0.002, 0.0005, 30)]), &cfg);
assert!(s.rolling_min_sharpe.is_none());
assert!(s.rolling_frac_positive.is_none());
}
#[test]
fn dsr_per_cost_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 60);
r.cost = 5.0;
let paid = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
let dpc = paid.dsr_per_cost.expect("reported with cost");
assert!((dpc - paid.deflated_sharpe / 5.0).abs() < 1e-12);
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 60)]),
&ScoreConfig::default(),
);
assert!(free.dsr_per_cost.is_none());
}
#[test]
fn process_violation_floors_realized_return() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.02, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("cheater", runs), &ScoreConfig::default());
assert!(s.process_floored, "block violation must set the floor flag");
assert_eq!(
s.realized_floored_return, 0.0,
"floored to no-skill baseline"
);
assert!(
s.raw_mean_return > 0.0,
"raw return is preserved un-floored"
);
assert!(!s.rank_eligible, "eligibility logic intact");
}
#[test]
fn clean_process_is_not_floored() {
let s = score_agent(
&agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(!s.process_floored);
assert_eq!(s.realized_floored_return, s.raw_mean_return);
}
#[test]
fn overlapping_dsr_cis_flag_a_tie_and_separation_gets_a_distinct_band() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let strong_a = agent("strong_a", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let strong_b = agent("strong_b", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let weak = agent("weak", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[strong_a, strong_b, weak], &cfg);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let (a, b, w) = (get("strong_a"), get("strong_b"), get("weak"));
assert!(
a.rank_eligible && b.rank_eligible && w.rank_eligible,
"all three should clear the (deliberately low) bar"
);
assert_eq!(a.tie_group, b.tie_group, "identical CIs share a band");
assert!(a.dsr_tied && b.dsr_tied, "the pair is flagged tied");
assert_ne!(
w.tie_group, a.tie_group,
"separable agent gets a distinct band"
);
assert!(!w.dsr_tied, "a distinct band is not a tie");
assert!(
w.dsr_ci_high < a.dsr_ci_low,
"weak CI upper {} should sit below strong CI lower {}",
w.dsr_ci_high,
a.dsr_ci_low
);
}
#[test]
fn rank_ordinal_is_one_based_among_eligible() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky, skilled], &ScoreConfig::default());
assert_eq!(board[0].rank_ordinal, 1, "leader is ordinal 1");
assert_eq!(board[1].rank_ordinal, 0, "ineligible gets ordinal 0");
}
#[test]
fn homogeneous_field_is_unchanged_by_shared_run_set() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let steady = agent("steady", (0..5).map(|_| run(0.001, 0.001, 60)).collect());
let field = [lucky, skilled, steady];
let on = rank(&field, &ScoreConfig::default());
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(on, off, "identical cells ⇒ identical board");
assert!(on
.iter()
.all(|s| s.runs_scored == 5 && s.runs_submitted == 5));
}
#[test]
fn easy_subset_entrant_is_compared_on_the_shared_cells() {
let easy = || run(0.004, 0.0005, 60);
let hard = || run(0.0, 0.004, 60);
let veteran = agent("veteran", vec![easy(), easy(), hard(), hard(), hard()]);
let entrant = agent("entrant", vec![easy(), easy()]);
let field = [veteran, entrant];
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(off[0].agent_id, "entrant");
assert!(off[0].rank_eligible && !off[1].rank_eligible);
let on = rank(&field, &ScoreConfig::default());
let get = |id: &str| on.iter().find(|s| s.agent_id == id).unwrap();
let (v, e) = (get("veteran"), get("entrant"));
assert_eq!(v.runs_scored, 2);
assert_eq!(e.runs_scored, 2);
assert_eq!(v.runs_submitted, 5);
assert_eq!(e.runs_submitted, 2);
assert_eq!(v.deflated_sharpe.to_bits(), e.deflated_sharpe.to_bits());
assert_eq!(v.rank_eligible, e.rank_eligible);
assert_eq!(v.tie_group, e.tie_group, "indistinguishable ⇒ one band");
assert!(v.dsr_tied && e.dsr_tied);
assert!(
e.rank_ordinal == 0 || v.rank_ordinal <= e.rank_ordinal,
"the entrant must not rank above the veteran"
);
}
#[test]
fn empty_submission_does_not_empty_the_shared_cells() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let ghost = agent("ghost", Vec::new());
let board = rank(&[ghost, skilled], &ScoreConfig::default());
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(get("skilled").runs_scored, 5);
assert!(get("skilled").rank_eligible);
assert!(!get("ghost").rank_eligible);
}
#[test]
fn small_field_keeps_the_configured_sr_std_byte_identical() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..4)
.map(|i| {
let m = 0.001 + 0.001 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.001, 60)).collect(),
)
})
.collect();
let board = rank(&field, &cfg);
assert_eq!(board.len(), 4);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), per_period_sr_std(&cfg).to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(cfg.trials_sr_std));
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
assert_eq!(s.deflated_sharpe.to_bits(), alone.deflated_sharpe.to_bits());
assert_eq!(s.dsr_ci_low.to_bits(), alone.dsr_ci_low.to_bits());
}
let pinned = rank(
&field,
&ScoreConfig {
min_field_for_measured_sr_std: usize::MAX,
..cfg
},
);
assert_eq!(board, pinned);
}
#[test]
fn large_field_measures_trials_sr_std_from_the_field() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let board = rank(&field, &cfg);
let mut sharpes: Vec<f64> = field
.iter()
.map(|a| {
let pooled: Vec<f64> = a
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
sharpe_ratio(&pooled)
})
.collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let expected = std_dev(&sharpes);
assert!(expected > 0.0 && expected.is_finite());
let mut moved = false;
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
moved |= s.deflated_sharpe.to_bits() != alone.deflated_sharpe.to_bits();
}
assert!(
moved,
"measured dispersion must actually move the deflation"
);
let mut reversed = field.clone();
reversed.reverse();
let again = rank(&reversed, &cfg);
assert_eq!(again[0].trials_sr_std.to_bits(), expected.to_bits());
}
fn gaussian_like(n: usize, mean_ret: f64, sd: f64, seed: u64) -> Vec<f64> {
let mut state = seed;
let mut next = || {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
(state >> 11) as f64 / (1u64 << 53) as f64
};
let raw: Vec<f64> = (0..n)
.map(|_| (0..12).map(|_| next()).sum::<f64>() - 6.0)
.collect();
let (m, s) = (mean(&raw), std_dev(&raw));
raw.iter().map(|x| mean_ret + sd * (x - m) / s).collect()
}
fn pooled_of(sub: &AgentSubmission) -> Vec<f64> {
sub.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
}
#[test]
fn annualized_prior_is_converted_per_period_once() {
let cfg = ScoreConfig {
trials_sr_std: 0.5,
periods_per_year: 8760.0,
..ScoreConfig::default()
};
let expected = 0.5 / 8760f64.sqrt();
assert_eq!(per_period_sr_std(&cfg).to_bits(), expected.to_bits());
let sub = agent("a", (0..5).map(|_| run(0.0002, 0.003, 300)).collect());
let s = score_agent(&sub, &cfg);
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(0.5));
let pooled = pooled_of(&sub);
let once = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected);
let raw = deflated_sharpe_ratio(&pooled, cfg.n_trials, 0.5);
let twice = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected / 8760f64.sqrt());
assert_eq!(s.deflated_sharpe.to_bits(), once.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), raw.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), twice.to_bits());
}
#[test]
fn clone_clusters_vote_once_on_the_measured_sr_std() {
let cfg = ScoreConfig::default();
let distinct: Vec<AgentSubmission> = (0..5)
.map(|i| {
agent(
&format!("a{i}"),
vec![Run {
returns: gaussian_like(120, 0.0002 + 0.0003 * i as f64, 0.003, 11 + i),
trace: Trace::default(),
confidences: vec![],
outcomes: vec![],
cost: 0.0,
}],
)
})
.collect();
let base = rank(&distinct, &cfg);
assert_eq!(base[0].trials_sr_std_source, TrialsSrStdSource::Measured);
let off = rank(
&distinct,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert_eq!(base, off, "no clones: the collapse is the identity");
let mut flooded = distinct.clone();
for k in 0..20 {
let mut copy = distinct[2].clone();
copy.agent_id = format!("copy{k:02}");
let lever = 1.0 + 0.05 * k as f64;
copy.runs[0].returns.iter_mut().for_each(|r| *r *= lever);
flooded.push(copy);
}
let board = rank(&flooded, &cfg);
assert_eq!(board.len(), 25, "clones are still scored");
assert_eq!(
board[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits(),
"twenty copies of one stream are one vote"
);
let mut shuffled = flooded.clone();
shuffled.rotate_left(7);
shuffled.swap(0, 12);
let again = rank(&shuffled, &cfg);
assert_eq!(
again[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits()
);
let exposed = rank(
&flooded,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert!(
exposed[0].trials_sr_std < base[0].trials_sr_std,
"without the collapse the copies shrink the dispersion"
);
}
#[test]
fn measured_sr_std_is_never_reconverted() {
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let mut sharpes: Vec<f64> = field.iter().map(|a| sharpe_ratio(&pooled_of(a))).collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let raw_measured = std_dev(&sharpes);
for ppy in [1.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
let board = rank(&field, &cfg);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), raw_measured.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let pooled = pooled_of(field.iter().find(|a| a.agent_id == s.agent_id).unwrap());
let expected = deflated_sharpe_ratio(&pooled, cfg.n_trials, raw_measured);
assert_eq!(s.deflated_sharpe.to_bits(), expected.to_bits());
}
}
}
#[test]
fn daily_buy_and_hold_clears_the_corrected_bar() {
let runs: Vec<Run> = (0..3)
.map(|i| Run {
returns: gaussian_like(2500, 0.0004, 0.011, 0x5B_A7 + i),
..Run::default()
})
.collect();
let index = agent("buy-and-hold", runs);
let cfg = ScoreConfig {
n_trials: 50,
trials_sr_std: 0.1,
periods_per_year: 252.0,
..ScoreConfig::default()
};
let s = score_agent(&index, &cfg);
assert!(s.psr > 0.99, "the index is clearly positive: {s:?}");
assert!(
s.rank_eligible,
"an index-like daily track must clear an annualized 0.1 prior: {s:?}"
);
let old_units = ScoreConfig {
periods_per_year: 1.0,
..cfg
};
let old = score_agent(&index, &old_units);
assert!(
!old.rank_eligible && old.deflated_sharpe < 0.05,
"applied per period the same prior was unreachable: {old:?}"
);
}
#[test]
fn hourly_bar_is_stricter_per_period_than_daily_for_the_same_annualized_prior() {
let daily = ScoreConfig::for_periods_per_year(252.0);
let hourly = ScoreConfig::for_periods_per_year(8760.0);
let star = |cfg: &ScoreConfig| expected_max_sharpe(per_period_sr_std(cfg), cfg.n_trials);
assert!(star(&hourly) > 0.0 && star(&daily) > 0.0);
assert!(
star(&hourly) < star(&daily),
"hourly sr_star {} must be below daily {}",
star(&hourly),
star(&daily)
);
let ann_h = star(&hourly) * 8760f64.sqrt();
let ann_d = star(&daily) * 252f64.sqrt();
assert!((ann_h - ann_d).abs() < 1e-12, "{ann_h} vs {ann_d}");
}
#[test]
fn default_min_annual_sharpe_is_identical_to_the_old_per_run_test() {
let mut runs = vec![run(0.02, 0.002, 60), run(0.0, 0.003, 60)];
runs.extend((0..3).map(|_| run(0.002, 0.0005, 60)));
let sub = agent("mixed", runs);
let clean = agent("clean", (0..3).map(|_| run(0.002, 0.0005, 60)).collect());
for ppy in [52.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
assert_eq!(per_run_psr_benchmark(&cfg).to_bits(), 0f64.to_bits());
let old: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, 0.0) >= cfg.per_run_psr_bar)
.collect();
assert!(old.iter().any(|&p| p) && old.iter().any(|&p| !p));
assert_eq!(
score_agent(&sub, &cfg).passed_k,
pass_k(&old, PassMode::All)
);
assert!(score_agent(&clean, &cfg).passed_k);
}
let strict = ScoreConfig {
per_run_min_annual_sharpe: 3.0,
..ScoreConfig::for_periods_per_year(252.0)
};
assert_eq!(
per_run_psr_benchmark(&strict).to_bits(),
(3.0 / 252f64.sqrt()).to_bits()
);
let weak = agent("weak", (0..3).map(|_| run(0.0005, 0.004, 60)).collect());
assert!(score_agent(&weak, &ScoreConfig::for_periods_per_year(252.0)).passed_k);
assert!(
!score_agent(&weak, &strict).passed_k,
"a minimum annualized Sharpe of 3.0 must fail a 0.0005/0.004 track"
);
}
fn field_with_one_single_run_failure() -> Vec<AgentSubmission> {
let clean = |id: &str| agent(id, (0..6).map(|_| run(0.002, 0.0005, 60)).collect());
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs.push(run(-0.002, 0.0005, 60));
vec![clean("a"), agent("one_bad_run", runs), clean("b")]
}
#[test]
fn default_pass_mode_is_all_and_matches_the_old_gate() {
let cfg = ScoreConfig::default();
assert_eq!(cfg.pass_mode, PassMode::All);
assert_eq!(cfg.mandate.max_run_drawdown, 1.0);
let field = field_with_one_single_run_failure();
let board = rank(&field, &cfg);
let benchmark = per_run_psr_benchmark(&cfg);
for s in &board {
let sub = field.iter().find(|a| a.agent_id == s.agent_id).unwrap();
let old_per_run: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, benchmark) >= cfg.per_run_psr_bar)
.collect();
let old_passed_k = pass_k(&old_per_run, PassMode::All);
assert_eq!(s.passed_k, old_passed_k, "{}", s.agent_id);
let old_eligible = s.deflated_sharpe >= cfg.dsr_bar
&& old_passed_k
&& s.process_ok
&& s.bootstrap_p < cfg.alpha
&& s.max_drawdown <= cfg.mandate.max_drawdown;
assert_eq!(s.rank_eligible, old_eligible, "{}", s.agent_id);
}
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert!(get("a").rank_eligible && get("b").rank_eligible);
assert!(!get("one_bad_run").rank_eligible);
assert!(!get("one_bad_run").passed_k);
}
#[test]
fn any_mode_admits_a_regime_dependent_edge_the_all_mode_rejects() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.003, 0.0005, 60)).collect();
runs.push(run(-0.001, 0.0005, 60));
let sub = agent("regime_edge", runs);
let all = score_agent(&sub, &ScoreConfig::default());
assert!(!all.passed_k && !all.rank_eligible, "{all:?}");
let any = score_agent(&sub, &ScoreConfig::reliability_never_catastrophic(0.20));
assert!(any.deflated_sharpe >= 0.95);
assert!(any.bootstrap_p < 0.05 && any.process_ok && any.mandate_ok);
assert!(any.worst_run_drawdown < 0.20 && any.max_drawdown < 0.20);
assert!(any.passed_k && any.rank_eligible, "{any:?}");
}
#[test]
fn per_run_drawdown_bound_rejects_one_catastrophic_run_that_the_pooled_bound_misses() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
for r in &mut crash.returns[10..15] {
*r = -0.03;
}
runs.push(crash);
let sub = agent("one_blowup", runs);
let loose_pooled = ScoreConfig {
mandate: Mandate {
max_drawdown: 0.20,
max_run_drawdown: 1.0,
},
pass_mode: PassMode::Any,
..ScoreConfig::default()
};
let pooled_only = score_agent(&sub, &loose_pooled);
assert!(pooled_only.max_drawdown <= 0.20, "{pooled_only:?}");
assert!(
pooled_only.mandate_ok && pooled_only.rank_eligible,
"{pooled_only:?}"
);
let mut preset = ScoreConfig::reliability_never_catastrophic(0.10);
preset.mandate.max_drawdown = 0.20;
let bounded = score_agent(&sub, &preset);
assert!(bounded.worst_run_drawdown > 0.10, "{bounded:?}");
assert!(bounded.max_drawdown <= 0.20, "the pooled bound still holds");
assert!(bounded.passed_k, "pass^k is not what rejects it");
assert!(!bounded.mandate_ok && !bounded.rank_eligible, "{bounded:?}");
}
#[test]
fn never_catastrophic_preset_is_weaker_than_the_default() {
let mut field = field_with_one_single_run_failure();
field.pop();
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
crash.returns[10] = -0.30;
runs.push(crash);
field.push(agent("blowup", runs));
let mut lucky = vec![run(0.02, 0.002, 60)];
lucky.extend((0..5).map(|_| run(0.0, 0.003, 60)));
field.push(agent("lucky", lucky));
assert!(field.iter().all(|a| a.runs.len() == 6));
assert!(field.len() < ScoreConfig::default().min_field_for_measured_sr_std);
let default = rank(&field, &ScoreConfig::default());
let preset = rank(&field, &ScoreConfig::reliability_never_catastrophic(0.20));
let eligible = |board: &[CompositeScore]| -> Vec<String> {
let mut v: Vec<String> = board
.iter()
.filter(|s| s.rank_eligible)
.map(|s| s.agent_id.clone())
.collect();
v.sort();
v
};
let (d, p) = (eligible(&default), eligible(&preset));
assert_eq!(d, vec!["a".to_string()], "{default:?}");
assert!(
d.iter().all(|id| p.contains(id)),
"default {d:?} not within preset {p:?}"
);
assert!(p.contains(&"one_bad_run".to_string()) && !d.contains(&"one_bad_run".to_string()));
assert!(
!p.contains(&"blowup".to_string()),
"a blow-up is refused under both"
);
assert!(
p.contains(&"lucky".to_string()) && !d.contains(&"lucky".to_string()),
"{preset:?}"
);
}
fn relative_field() -> Vec<AgentSubmission> {
let bench: Vec<Run> = (0..5)
.map(|_| run(0.002, 0.0005, 60))
.chain(std::iter::once(run(-0.003, 0.0005, 60)))
.collect();
let beats: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for (i, x) in r.returns.iter_mut().enumerate() {
*x += 0.001 + 0.0002 * (i as f64 * 0.7).sin();
}
r
})
.collect();
let mut five_of_six = beats.clone();
for (i, x) in five_of_six[5].returns.iter_mut().enumerate() {
*x = bench[5].returns[i] - 0.001;
}
let trails: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for x in r.returns.iter_mut() {
*x = x.abs() * 0.5 + 0.0005;
}
r
})
.collect();
vec![
agent("buy-and-hold", bench.clone()),
agent("beats", beats),
agent("five-of-six", five_of_six),
agent("trails", trails),
agent("clone", bench),
]
}
#[test]
fn default_verdict_ignores_the_benchmark_id() {
let field = relative_field();
let default = rank(&field, &ScoreConfig::default());
let renamed = rank(
&field,
&ScoreConfig {
benchmark_agent_id: "no-such-agent".to_string(),
..ScoreConfig::default()
},
);
assert_eq!(default, renamed);
assert_eq!(
serde_json::from_str::<ScoreConfig>("{\"n_trials\":1,\"trials_sr_std\":0.5,\"dsr_bar\":0.9,\"per_run_psr_bar\":0.9,\"alpha\":0.05,\"bootstrap_seed\":1,\"n_boot\":10,\"block_prob\":0.1}")
.unwrap()
.benchmark_agent_id,
"buy-and-hold"
);
}
#[test]
fn relative_verdict_tests_excess_over_the_same_cell() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("buy-and-hold");
let default = rank(&field, &ScoreConfig::default());
let relative = rank(&field, &cfg);
let passed = |board: &[CompositeScore], id: &str| {
board.iter().find(|s| s.agent_id == id).unwrap().passed_k
};
assert!(!passed(&default, "beats") && passed(&relative, "beats"));
assert!(passed(&default, "trails") && !passed(&relative, "trails"));
assert!(!passed(&default, "buy-and-hold") && !passed(&relative, "buy-and-hold"));
assert!(!passed(&relative, "clone"));
assert!(!passed(&relative, "five-of-six"));
let bench = &field[0];
let five = per_run_passes(&field[2], Some(bench), &cfg);
assert_eq!(five, vec![true, true, true, true, true, false]);
let zero = per_run_passes(bench, Some(bench), &cfg);
assert_eq!(zero, vec![false; 6]);
let lax = ScoreConfig {
per_run_psr_bar: 0.5,
..cfg.clone()
};
assert_eq!(per_run_passes(bench, Some(bench), &lax), vec![false; 6]);
for d in &default {
let r = relative.iter().find(|s| s.agent_id == d.agent_id).unwrap();
assert_eq!(d.deflated_sharpe.to_bits(), r.deflated_sharpe.to_bits());
assert_eq!(d.psr.to_bits(), r.psr.to_bits());
assert_eq!(d.bootstrap_p.to_bits(), r.bootstrap_p.to_bits());
assert_eq!(
d.worst_run_drawdown.to_bits(),
r.worst_run_drawdown.to_bits()
);
assert_eq!(d.process_ok, r.process_ok);
}
}
#[test]
fn relative_verdict_without_a_benchmark_fails_every_run() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("absent");
assert!(rank(&field, &cfg).iter().all(|s| !s.passed_k));
let alone = score_agent(
&field[1],
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert!(!alone.passed_k);
let mut short = field[0].clone();
short.runs[2].returns.pop();
let v = per_run_passes(
&field[1],
Some(&short),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, false, true, true, true]);
let mut fewer = field[0].clone();
fewer.runs.truncate(4);
let v = per_run_passes(
&field[1],
Some(&fewer),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, true, true, false, false]);
}
#[test]
fn relative_preset_differs_only_in_mode_and_benchmark_id() {
let mut preset = serde_json::to_value(ScoreConfig::relative_to_benchmark("index")).unwrap();
let default = serde_json::to_value(ScoreConfig::default()).unwrap();
assert_eq!(preset["pass_mode"], "relative_to_benchmark");
assert_eq!(preset["benchmark_agent_id"], "index");
preset["pass_mode"] = default["pass_mode"].clone();
preset["benchmark_agent_id"] = default["benchmark_agent_id"].clone();
assert_eq!(preset, default);
}
#[test]
fn warn_events_lower_process_score_and_count_but_not_eligibility() {
let clean = agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let mut warned_runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
warned_runs[3]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
let cfg = ScoreConfig::default();
let c = score_agent(&clean, &cfg);
let w = score_agent(&warned, &cfg);
assert_eq!(c.process_warnings, 0);
assert_eq!(c.process_score, 1.0);
assert_eq!(w.process_warnings, 2);
assert!((w.process_score - 0.8).abs() < 1e-9);
assert!(c.process_ok && w.process_ok);
assert_eq!(c.rank_eligible, w.rank_eligible);
assert!(w.rank_eligible);
assert_eq!(c.deflated_sharpe.to_bits(), w.deflated_sharpe.to_bits());
assert_eq!(c.composite.to_bits(), w.composite.to_bits());
}
#[test]
fn block_violation_zeroes_graded_process_score() {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
runs[1].trace.events.push(ProcessEvent::DenylistBypass);
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
let s = score_agent(&agent("blocked", runs), &ScoreConfig::default());
assert_eq!(s.process_score, 0.0);
assert_eq!(s.process_warnings, 1, "warns are still counted");
assert!(!s.process_ok && !s.rank_eligible);
}
#[test]
fn tie_band_orders_cleaner_process_first() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let clean = agent("clean", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let mut warned_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
for field in [
vec![clean.clone(), warned.clone()],
vec![warned.clone(), clean.clone()],
] {
let board = rank(&field, &cfg);
assert_eq!(board[0].agent_id, "clean", "cleaner process leads the band");
assert_eq!(board[1].agent_id, "warned");
assert_eq!(board[0].rank_ordinal, 1);
assert_eq!(board[1].rank_ordinal, 2);
assert!(board[0].rank_eligible && board[1].rank_eligible);
assert_eq!(board[0].tie_group, board[1].tie_group);
assert!(board[0].dsr_tied && board[1].dsr_tied);
}
}
#[test]
fn process_tie_break_never_crosses_tie_bands() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let mut strong_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
strong_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let strong_warned = agent("strong_warned", strong_runs);
let weak_clean = agent("weak_clean", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[weak_clean, strong_warned], &cfg);
assert!(board.iter().all(|s| s.rank_eligible));
assert_eq!(board[0].agent_id, "strong_warned");
assert_ne!(board[0].tie_group, board[1].tie_group, "separable bands");
assert!(board[0].process_score < board[1].process_score);
}
#[test]
fn econ_rationality_reported_from_declared_candidates() {
let cfg = ScoreConfig::default();
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let none = score_agent(&agent("plain", runs.clone()), &cfg);
assert!(none.econ_rationality_score.is_none());
assert!(none.econ_dominance_violations.is_none());
let mut dominated = agent("dominated", runs.clone());
dominated.candidates = vec![(0..300)
.map(|i| 0.01 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let d = score_agent(&dominated, &cfg);
assert_eq!(d.econ_rationality_score, Some(0.0));
assert_eq!(d.econ_dominance_violations, Some(1));
let mut rational = agent("rational", runs);
rational.candidates = vec![(0..300)
.map(|i| 0.0001 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let r = score_agent(&rational, &cfg);
assert_eq!(r.econ_rationality_score, Some(1.0));
assert_eq!(r.econ_dominance_violations, Some(0));
assert_eq!(d.rank_eligible, r.rank_eligible);
}
#[test]
fn behavior_role_contributions_are_reported() {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
for r in &mut runs {
r.trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: true,
});
}
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
let s = score_agent(&agent("mixed", runs), &ScoreConfig::default());
let names: Vec<&str> = s
.role_contributions
.iter()
.map(|c| c.role.as_str())
.collect();
assert_eq!(names, vec!["clean_active", "warned"]);
let again = score_agent(
&agent("mixed", {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
for r in &mut runs {
r.trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: true,
});
}
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
runs
}),
&ScoreConfig::default(),
);
assert_eq!(s.role_contributions, again.role_contributions);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert!(empty.role_contributions.is_empty());
}
#[test]
fn archived_scores_without_new_fields_deserialize_with_defaults() {
let s = score_agent(
&agent("archived", (0..3).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
let mut v = serde_json::to_value(&s).expect("serialize");
let obj = v.as_object_mut().expect("object");
for field in [
"process_score",
"process_warnings",
"econ_rationality_score",
"econ_dominance_violations",
"role_contributions",
] {
assert!(obj.remove(field).is_some(), "{field} should be present");
}
let parsed: CompositeScore = serde_json::from_value(v).expect("archived JSON parses");
assert_eq!(parsed.process_score, 1.0);
assert_eq!(parsed.process_warnings, 0);
assert!(parsed.econ_rationality_score.is_none());
assert!(parsed.econ_dominance_violations.is_none());
assert!(parsed.role_contributions.is_empty());
}
#[test]
fn worst_run_drawdown_is_the_max_over_runs_not_the_pooled_track() {
let down = || Run {
returns: vec![-0.05; 4],
..Run::default()
};
let up = Run {
returns: vec![0.10; 10],
..Run::default()
};
let sub = agent("runs", vec![up.clone(), down(), down()]);
let s = score_agent(&sub, &ScoreConfig::default());
let expected = max_drawdown(&up.returns).max(max_drawdown(&down().returns));
assert_eq!(s.worst_run_drawdown.to_bits(), expected.to_bits());
assert!((s.worst_run_drawdown - (1.0 - 0.95f64.powi(4))).abs() < 1e-12);
assert_eq!(
s.max_drawdown.to_bits(),
max_drawdown(&pooled_of(&sub)).to_bits()
);
assert!((s.max_drawdown - (1.0 - 0.95f64.powi(8))).abs() < 1e-12);
assert!(s.max_drawdown > s.worst_run_drawdown);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert_eq!(empty.worst_run_drawdown, 0.0);
}
}