use serde::{Deserialize, Serialize};
use crate::calibration::brier_score;
use crate::comparison_sets::{comparison_set, restrict_to_shared, TaggedRun, TaggedSubmission};
use crate::decay::edge_half_life;
use crate::deflated_sharpe::{deflated_sharpe_ratio, probabilistic_sharpe_ratio, sharpe_ratio};
use crate::pass_k::{pass_k, PassMode};
use crate::percentile::percentile_of;
use crate::process::{process_score, ProcessEvent, Trace};
use crate::rolling::rolling_sharpe;
use crate::selection::{selection_robustness, SelectionRobustness};
use crate::significance::bootstrap_pvalue;
use crate::stats::{mean, std_dev};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Run {
pub returns: Vec<f64>,
#[serde(default)]
pub trace: Trace,
#[serde(default)]
pub confidences: Vec<f64>,
#[serde(default)]
pub outcomes: Vec<bool>,
#[serde(default)]
pub cost: f64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct AgentSubmission {
pub agent_id: String,
pub runs: Vec<Run>,
#[serde(default)]
pub in_sample_trials: u32,
#[serde(default)]
pub candidates: Vec<Vec<f64>>,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub enum RankKey {
#[default]
DeflatedSharpe,
Alpha,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Mandate {
pub max_drawdown: f64,
#[serde(default = "default_max_run_drawdown")]
pub max_run_drawdown: f64,
}
impl Default for Mandate {
fn default() -> Self {
Self {
max_drawdown: 1.0,
max_run_drawdown: default_max_run_drawdown(),
}
}
}
fn default_max_run_drawdown() -> f64 {
1.0
}
fn max_drawdown(returns: &[f64]) -> f64 {
let mut nav = 1.0;
let mut peak = 1.0;
let mut mdd = 0.0;
for &r in returns {
nav *= 1.0 + r;
if nav > peak {
peak = nav;
}
if peak > 0.0 {
let dd = 1.0 - nav / peak;
if dd > mdd {
mdd = dd;
}
}
}
mdd
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreConfig {
pub n_trials: u32,
pub trials_sr_std: f64,
pub dsr_bar: f64,
pub per_run_psr_bar: f64,
#[serde(default)]
pub per_run_min_annual_sharpe: f64,
#[serde(default)]
pub pass_mode: PassMode,
#[serde(default = "default_periods_per_year")]
pub periods_per_year: f64,
pub alpha: f64,
pub bootstrap_seed: u64,
pub n_boot: usize,
pub block_prob: f64,
#[serde(default)]
pub mandate: Mandate,
#[serde(default)]
pub rank_key: RankKey,
#[serde(default)]
pub reference_dsr_population: Vec<f64>,
#[serde(default = "default_rolling_window")]
pub rolling_window: usize,
#[serde(default = "default_dsr_ci_level")]
pub dsr_ci_level: f64,
#[serde(default = "default_shared_run_set")]
pub shared_run_set: bool,
#[serde(default = "default_min_field_for_measured_sr_std")]
pub min_field_for_measured_sr_std: usize,
}
fn default_rolling_window() -> usize {
21
}
fn default_dsr_ci_level() -> f64 {
0.90
}
fn default_shared_run_set() -> bool {
true
}
fn default_min_field_for_measured_sr_std() -> usize {
5
}
fn default_periods_per_year() -> f64 {
252.0
}
pub fn per_period_sr_std(cfg: &ScoreConfig) -> f64 {
cfg.trials_sr_std / cfg.periods_per_year.sqrt()
}
pub fn per_run_psr_benchmark(cfg: &ScoreConfig) -> f64 {
cfg.per_run_min_annual_sharpe / cfg.periods_per_year.sqrt()
}
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TrialsSrStdSource {
#[default]
Configured,
Measured,
}
impl Default for ScoreConfig {
fn default() -> Self {
Self {
n_trials: 50,
trials_sr_std: 0.5,
dsr_bar: 0.95,
per_run_psr_bar: 0.90,
per_run_min_annual_sharpe: 0.0,
pass_mode: PassMode::default(),
periods_per_year: default_periods_per_year(),
alpha: 0.05,
bootstrap_seed: 0x5BA7_2026,
n_boot: 2000,
block_prob: 0.1,
mandate: Mandate::default(),
rank_key: RankKey::default(),
reference_dsr_population: Vec::new(),
rolling_window: default_rolling_window(),
dsr_ci_level: default_dsr_ci_level(),
shared_run_set: default_shared_run_set(),
min_field_for_measured_sr_std: default_min_field_for_measured_sr_std(),
}
}
}
impl ScoreConfig {
pub fn for_periods_per_year(periods_per_year: f64) -> Self {
Self {
periods_per_year,
..Self::default()
}
}
pub fn reliability_never_catastrophic(max_run_dd: f64) -> Self {
Self {
pass_mode: PassMode::Any,
mandate: Mandate {
max_run_drawdown: max_run_dd,
..Mandate::default()
},
..Self::default()
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct CompositeScore {
pub agent_id: String,
pub deflated_sharpe: f64,
pub psr: f64,
pub passed_k: bool,
pub process_ok: bool,
pub bootstrap_p: f64,
pub raw_mean_return: f64,
pub rank_eligible: bool,
pub composite: f64,
pub alpha: f64,
pub beta: f64,
pub calibration_brier: Option<f64>,
pub edge_half_life: Option<f64>,
pub field_reality_check_p: f64,
pub max_drawdown: f64,
pub mandate_ok: bool,
#[serde(default)]
pub worst_run_drawdown: f64,
pub turnover: f64,
pub pareto_optimal: bool,
pub step_down_significant: bool,
pub confidence_weighted_return: f64,
pub cost: f64,
pub return_per_cost: Option<f64>,
pub field_spa_p: f64,
pub field_spa_consistent_p: f64,
pub field_crowdedness: Option<f64>,
pub in_sample_trials: u32,
pub effective_n_trials: u32,
pub dsr_percentile: Option<f64>,
pub selection_median_dsr: Option<f64>,
pub selection_gap: Option<f64>,
pub rank_ordinal: usize,
pub rolling_min_sharpe: Option<f64>,
pub rolling_frac_positive: Option<f64>,
pub sortino: Option<f64>,
pub downside_deviation: f64,
pub dsr_per_cost: Option<f64>,
pub process_floored: bool,
pub realized_floored_return: f64,
pub dsr_ci_low: f64,
pub dsr_ci_high: f64,
pub dsr_se: f64,
pub tie_group: usize,
pub dsr_tied: bool,
#[serde(default)]
pub trials_sr_std: f64,
#[serde(default)]
pub trials_sr_std_annualized: Option<f64>,
#[serde(default)]
pub trials_sr_std_source: TrialsSrStdSource,
#[serde(default)]
pub runs_submitted: usize,
#[serde(default)]
pub runs_scored: usize,
}
fn dominates(a: &CompositeScore, b: &CompositeScore) -> bool {
a.raw_mean_return >= b.raw_mean_return
&& a.max_drawdown <= b.max_drawdown
&& a.turnover <= b.turnover
&& (a.raw_mean_return > b.raw_mean_return
|| a.max_drawdown < b.max_drawdown
|| a.turnover < b.turnover)
}
#[derive(Clone, Copy)]
struct Deflation {
sr_std: f64,
annualized: Option<f64>,
source: TrialsSrStdSource,
}
impl Deflation {
fn configured(cfg: &ScoreConfig) -> Self {
Self {
sr_std: per_period_sr_std(cfg),
annualized: Some(cfg.trials_sr_std),
source: TrialsSrStdSource::Configured,
}
}
fn measured(sr_std: f64) -> Self {
Self {
sr_std,
annualized: None,
source: TrialsSrStdSource::Measured,
}
}
}
pub fn score_agent(sub: &AgentSubmission, cfg: &ScoreConfig) -> CompositeScore {
score_agent_with(sub, cfg, Deflation::configured(cfg))
}
fn score_agent_with(sub: &AgentSubmission, cfg: &ScoreConfig, defl: Deflation) -> CompositeScore {
let pooled: Vec<f64> = sub
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
let psr = probabilistic_sharpe_ratio(&pooled, 0.0);
let effective_n_trials = cfg.n_trials.saturating_add(sub.in_sample_trials);
let dsr = deflated_sharpe_ratio(&pooled, effective_n_trials, defl.sr_std);
let per_run_benchmark = per_run_psr_benchmark(cfg);
let per_run: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, per_run_benchmark) >= cfg.per_run_psr_bar)
.collect();
let passed_k = pass_k(&per_run, cfg.pass_mode);
let process_ok = sub.runs.iter().all(|r| process_score(&r.trace).is_clean());
let bootstrap_p = bootstrap_pvalue(&pooled, cfg.bootstrap_seed, cfg.n_boot, cfg.block_prob);
let raw_mean_return = mean(&pooled);
let conf: Vec<f64> = sub
.runs
.iter()
.flat_map(|r| r.confidences.iter().copied())
.collect();
let outc: Vec<bool> = sub
.runs
.iter()
.flat_map(|r| r.outcomes.iter().copied())
.collect();
let calibration_brier = if !conf.is_empty() && !outc.is_empty() {
Some(brier_score(&conf, &outc))
} else {
None
};
let per_run_edge: Vec<f64> = sub.runs.iter().map(|r| mean(&r.returns)).collect();
let edge_half_life_periods = edge_half_life(&per_run_edge);
let mdd = max_drawdown(&pooled);
let worst_run_drawdown = sub
.runs
.iter()
.map(|r| max_drawdown(&r.returns))
.fold(0.0, f64::max);
let mandate_ok =
mdd <= cfg.mandate.max_drawdown && worst_run_drawdown <= cfg.mandate.max_run_drawdown;
let total_orders: usize = sub
.runs
.iter()
.map(|r| {
r.trace
.events
.iter()
.filter(|e| matches!(e, ProcessEvent::OrderPlaced { .. }))
.count()
})
.sum();
let turnover = total_orders as f64 / sub.runs.len().max(1) as f64;
let mut cw_num = 0.0;
let mut cw_den = 0.0;
for r in &sub.runs {
let w = if r.confidences.is_empty() {
1.0
} else {
mean(&r.confidences)
};
cw_num += w * mean(&r.returns);
cw_den += w;
}
let confidence_weighted_return = if cw_den > 0.0 {
cw_num / cw_den
} else {
raw_mean_return
};
let cost: f64 = sub.runs.iter().map(|r| r.cost).sum();
let return_per_cost = if cost > 0.0 {
Some(raw_mean_return / cost)
} else {
None
};
let dsr_percentile = if cfg.reference_dsr_population.is_empty() {
None
} else {
Some(percentile_of(dsr, &cfg.reference_dsr_population))
};
let (selection_median_dsr, selection_gap) = if sub.candidates.is_empty() {
(None, None)
} else {
let sr: SelectionRobustness =
selection_robustness(&sub.candidates, effective_n_trials, defl.sr_std);
(Some(sr.median_dsr), Some(sr.selection_gap))
};
let rolling = rolling_sharpe(&pooled, cfg.rolling_window);
let rolling_min_sharpe = rolling.map(|r| r.min_sharpe);
let rolling_frac_positive = rolling.map(|r| r.frac_positive);
let sortino = crate::stats::sortino_ratio(&pooled, 0.0);
let downside_deviation = crate::stats::downside_deviation(&pooled, 0.0);
let dsr_per_cost = if cost > 0.0 { Some(dsr / cost) } else { None };
let process_floored = !process_ok;
let realized_floored_return = if process_floored {
0.0
} else {
raw_mean_return
};
let dsr_ci = crate::significance::bootstrap_dsr_ci(
&pooled,
effective_n_trials,
defl.sr_std,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.dsr_ci_level,
);
let rank_eligible =
dsr >= cfg.dsr_bar && passed_k && process_ok && bootstrap_p < cfg.alpha && mandate_ok;
let composite = if rank_eligible { dsr } else { 0.0 };
CompositeScore {
agent_id: sub.agent_id.clone(),
deflated_sharpe: dsr,
psr,
passed_k,
process_ok,
bootstrap_p,
raw_mean_return,
rank_eligible,
composite,
alpha: 0.0,
beta: 0.0,
calibration_brier,
edge_half_life: edge_half_life_periods,
field_reality_check_p: 1.0,
max_drawdown: mdd,
mandate_ok,
worst_run_drawdown,
turnover,
pareto_optimal: false,
step_down_significant: false,
confidence_weighted_return,
cost,
return_per_cost,
field_spa_p: 1.0,
field_spa_consistent_p: 1.0,
field_crowdedness: None,
in_sample_trials: sub.in_sample_trials,
effective_n_trials,
dsr_percentile,
selection_median_dsr,
selection_gap,
rank_ordinal: 0,
rolling_min_sharpe,
rolling_frac_positive,
sortino,
downside_deviation,
dsr_per_cost,
process_floored,
realized_floored_return,
dsr_ci_low: dsr_ci.lower,
dsr_ci_high: dsr_ci.upper,
dsr_se: dsr_ci.se,
tie_group: 0,
dsr_tied: false,
trials_sr_std: defl.sr_std,
trials_sr_std_annualized: defl.annualized,
trials_sr_std_source: defl.source,
runs_submitted: sub.runs.len(),
runs_scored: sub.runs.len(),
}
}
fn restrict_to_shared_positions(subs: &[AgentSubmission]) -> Vec<AgentSubmission> {
let tag = |i: usize| format!("{i:08}");
let tagged: Vec<TaggedSubmission> = subs
.iter()
.filter(|s| !s.runs.is_empty())
.map(|s| TaggedSubmission {
agent_id: s.agent_id.clone(),
runs: s
.runs
.iter()
.enumerate()
.map(|(i, run)| TaggedRun {
window_id: tag(i),
run: run.clone(),
})
.collect(),
in_sample_trials: s.in_sample_trials,
candidates: s.candidates.clone(),
})
.collect();
let roster: Vec<String> = tagged.iter().map(|s| s.agent_id.clone()).collect();
let set = comparison_set(&roster, &tagged);
let mut tagged_iter = tagged.iter();
subs.iter()
.map(|s| {
if s.runs.is_empty() {
return s.clone();
}
let t = tagged_iter
.next()
.expect("one tagged submission per non-empty submission");
restrict_to_shared(&set, t)
})
.collect()
}
fn measured_trials_sr_std(pooled: &[Vec<f64>], min_field: usize) -> Option<f64> {
let mut sharpes: Vec<f64> = pooled
.iter()
.filter(|p| p.len() >= 2)
.map(|p| sharpe_ratio(p))
.filter(|sr| sr.is_finite())
.collect();
if sharpes.len() < min_field.max(2) {
return None;
}
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
Some(std_dev(&sharpes))
}
pub fn rank(subs: &[AgentSubmission], cfg: &ScoreConfig) -> Vec<CompositeScore> {
let restricted;
let field: &[AgentSubmission] = if cfg.shared_run_set {
restricted = restrict_to_shared_positions(subs);
&restricted
} else {
subs
};
let pooled: Vec<Vec<f64>> = field
.iter()
.map(|s| {
s.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
})
.collect();
let min_len = pooled.iter().map(Vec::len).min().unwrap_or(0);
let n_agents = pooled.len().max(1) as f64;
let market: Vec<f64> = (0..min_len)
.map(|i| pooled.iter().map(|p| p[i]).sum::<f64>() / n_agents)
.collect();
let defl = measured_trials_sr_std(&pooled, cfg.min_field_for_measured_sr_std)
.map_or_else(|| Deflation::configured(cfg), Deflation::measured);
let mut scores: Vec<CompositeScore> = field
.iter()
.enumerate()
.map(|(idx, s)| {
let mut cs = score_agent_with(s, cfg, defl);
cs.runs_submitted = subs[idx].runs.len();
if min_len >= 2 {
let (alpha, beta) = crate::attribution::alpha_beta(&pooled[idx], &market);
cs.alpha = alpha;
cs.beta = beta;
}
cs
})
.collect();
if min_len >= 2 {
let field_excess: Vec<Vec<f64>> = pooled
.iter()
.map(|p| {
p.iter()
.take(min_len)
.zip(market.iter())
.map(|(a, m)| a - m)
.collect()
})
.collect();
let rc_p = crate::significance::reality_check_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_p = crate::significance::spa_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_c_p = crate::significance::spa_consistent_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
for cs in scores.iter_mut() {
cs.field_reality_check_p = rc_p;
cs.field_spa_p = spa_p;
cs.field_spa_consistent_p = spa_c_p;
}
let sd = crate::significance::step_down_significant(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.alpha,
);
for (cs, s) in scores.iter_mut().zip(sd) {
cs.step_down_significant = s;
}
}
if min_len >= 2 && pooled.len() >= 2 {
let aligned: Vec<&[f64]> = pooled.iter().map(|p| &p[..min_len]).collect();
for (idx, cs) in scores.iter_mut().enumerate() {
let peers: Vec<&[f64]> = aligned
.iter()
.enumerate()
.filter(|&(j, _)| j != idx)
.map(|(_, &p)| p)
.collect();
cs.field_crowdedness = crate::correlation::crowdedness(aligned[idx], &peers).mean_corr;
}
}
let pareto: Vec<bool> = (0..scores.len())
.map(|i| !(0..scores.len()).any(|j| j != i && dominates(&scores[j], &scores[i])))
.collect();
for (cs, p) in scores.iter_mut().zip(pareto) {
cs.pareto_optimal = p;
}
let sort_key = |s: &CompositeScore| match cfg.rank_key {
RankKey::DeflatedSharpe => s.composite,
RankKey::Alpha => {
if s.rank_eligible {
s.alpha
} else {
f64::NEG_INFINITY
}
}
};
scores.sort_by(|a, b| {
b.rank_eligible
.cmp(&a.rank_eligible)
.then(
sort_key(b)
.partial_cmp(&sort_key(a))
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
b.raw_mean_return
.partial_cmp(&a.raw_mean_return)
.unwrap_or(std::cmp::Ordering::Equal),
)
});
let mut ord = 0usize;
for cs in scores.iter_mut() {
if cs.rank_eligible {
ord += 1;
cs.rank_ordinal = ord;
}
}
let mut group = 0usize;
let mut prev: Option<usize> = None;
for i in 0..scores.len() {
if !scores[i].rank_eligible {
continue;
}
let same_band = matches!(prev, Some(p) if ci_overlap(&scores[p], &scores[i]));
if !same_band {
group += 1;
}
scores[i].tie_group = group;
prev = Some(i);
}
let mut band_counts = vec![0usize; group + 1];
for cs in &scores {
if cs.rank_eligible {
band_counts[cs.tie_group] += 1;
}
}
for cs in scores.iter_mut() {
if cs.rank_eligible {
cs.dsr_tied = band_counts[cs.tie_group] > 1;
}
}
scores
}
fn ci_overlap(a: &CompositeScore, b: &CompositeScore) -> bool {
a.dsr_ci_low <= b.dsr_ci_high && b.dsr_ci_low <= a.dsr_ci_high
}
#[cfg(test)]
mod tests {
use super::*;
use crate::deflated_sharpe::expected_max_sharpe;
use crate::process::ProcessEvent;
fn run(mean_ret: f64, amp: f64, n: usize) -> Run {
let returns = (0..n)
.map(|i| mean_ret + amp * (i as f64 * 0.7).sin())
.collect();
Run {
returns,
trace: Trace::default(),
confidences: Vec::new(),
outcomes: Vec::new(),
cost: 0.0,
}
}
fn agent(id: &str, runs: Vec<Run>) -> AgentSubmission {
AgentSubmission {
agent_id: id.to_string(),
runs,
in_sample_trials: 0,
candidates: Vec::new(),
}
}
#[test]
fn skilled_is_eligible() {
let s = score_agent(
&agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rank_eligible, "skilled should be eligible: {s:?}");
assert!(s.passed_k && s.process_ok);
}
#[test]
fn lucky_high_return_fails_pass_k() {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
let s = score_agent(&agent("lucky", runs), &ScoreConfig::default());
assert!(!s.passed_k, "lucky should fail pass^k");
assert!(!s.rank_eligible, "lucky must not be rank-eligible: {s:?}");
}
#[test]
fn process_violator_is_disqualified() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("violator", runs), &ScoreConfig::default());
assert!(!s.process_ok);
assert!(!s.rank_eligible, "a risk-gate bypass must disqualify");
}
#[test]
fn deflation_demotes_luck() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky.clone(), skilled.clone()], &ScoreConfig::default());
let lucky_raw = board
.iter()
.find(|s| s.agent_id == "lucky")
.unwrap()
.raw_mean_return;
let skilled_raw = board
.iter()
.find(|s| s.agent_id == "skilled")
.unwrap()
.raw_mean_return;
assert!(
lucky_raw > skilled_raw,
"lucky raw {lucky_raw} should exceed skilled {skilled_raw}"
);
assert_eq!(board[0].agent_id, "skilled");
assert!(board[0].rank_eligible && !board[1].rank_eligible);
}
#[test]
fn confidence_weighting_rewards_conviction() {
let win = Run {
returns: vec![0.01; 30],
trace: Trace::default(),
confidences: vec![0.9; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let lose = Run {
returns: vec![-0.005; 30],
trace: Trace::default(),
confidences: vec![0.1; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let s = score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default());
assert!(
s.confidence_weighted_return > s.raw_mean_return,
"cwr {} should beat raw {}",
s.confidence_weighted_return,
s.raw_mean_return
);
}
#[test]
fn cost_efficiency_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 30);
r.cost = 4.0;
let s = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
assert_eq!(s.cost, 4.0);
assert!(s.return_per_cost.is_some());
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 30)]),
&ScoreConfig::default(),
);
assert!(free.return_per_cost.is_none());
}
#[test]
fn in_sample_search_raises_the_deflation_bar() {
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let base = score_agent(&agent("base", runs.clone()), &ScoreConfig::default());
let mut over = agent("over", runs);
over.in_sample_trials = 5000;
let s = score_agent(&over, &ScoreConfig::default());
assert_eq!(s.effective_n_trials, 5050);
assert!(
s.deflated_sharpe <= base.deflated_sharpe,
"more in-sample search must not raise DSR ({} vs {})",
s.deflated_sharpe,
base.deflated_sharpe
);
}
#[test]
fn percentile_reported_only_with_reference() {
let none = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(none.dsr_percentile.is_none());
let cfg = ScoreConfig {
reference_dsr_population: vec![0.0, 0.3, 0.6, 0.9],
..ScoreConfig::default()
};
let some = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&cfg,
);
assert!(some.dsr_percentile.is_some());
}
#[test]
fn rolling_sharpe_reported_for_long_tracks() {
let s = score_agent(
&agent("roll", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rolling_min_sharpe.is_some());
let fp = s.rolling_frac_positive.expect("reported");
assert!(
(fp - 1.0).abs() < 1e-12,
"steady edge → all windows positive"
);
}
#[test]
fn rolling_sharpe_none_when_track_too_short() {
let cfg = ScoreConfig {
rolling_window: 100,
..ScoreConfig::default()
};
let s = score_agent(&agent("short", vec![run(0.002, 0.0005, 30)]), &cfg);
assert!(s.rolling_min_sharpe.is_none());
assert!(s.rolling_frac_positive.is_none());
}
#[test]
fn dsr_per_cost_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 60);
r.cost = 5.0;
let paid = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
let dpc = paid.dsr_per_cost.expect("reported with cost");
assert!((dpc - paid.deflated_sharpe / 5.0).abs() < 1e-12);
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 60)]),
&ScoreConfig::default(),
);
assert!(free.dsr_per_cost.is_none());
}
#[test]
fn process_violation_floors_realized_return() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.02, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("cheater", runs), &ScoreConfig::default());
assert!(s.process_floored, "block violation must set the floor flag");
assert_eq!(
s.realized_floored_return, 0.0,
"floored to no-skill baseline"
);
assert!(
s.raw_mean_return > 0.0,
"raw return is preserved un-floored"
);
assert!(!s.rank_eligible, "eligibility logic intact");
}
#[test]
fn clean_process_is_not_floored() {
let s = score_agent(
&agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(!s.process_floored);
assert_eq!(s.realized_floored_return, s.raw_mean_return);
}
#[test]
fn overlapping_dsr_cis_flag_a_tie_and_separation_gets_a_distinct_band() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let strong_a = agent("strong_a", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let strong_b = agent("strong_b", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let weak = agent("weak", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[strong_a, strong_b, weak], &cfg);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let (a, b, w) = (get("strong_a"), get("strong_b"), get("weak"));
assert!(
a.rank_eligible && b.rank_eligible && w.rank_eligible,
"all three should clear the (deliberately low) bar"
);
assert_eq!(a.tie_group, b.tie_group, "identical CIs share a band");
assert!(a.dsr_tied && b.dsr_tied, "the pair is flagged tied");
assert_ne!(
w.tie_group, a.tie_group,
"separable agent gets a distinct band"
);
assert!(!w.dsr_tied, "a distinct band is not a tie");
assert!(
w.dsr_ci_high < a.dsr_ci_low,
"weak CI upper {} should sit below strong CI lower {}",
w.dsr_ci_high,
a.dsr_ci_low
);
}
#[test]
fn rank_ordinal_is_one_based_among_eligible() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky, skilled], &ScoreConfig::default());
assert_eq!(board[0].rank_ordinal, 1, "leader is ordinal 1");
assert_eq!(board[1].rank_ordinal, 0, "ineligible gets ordinal 0");
}
#[test]
fn homogeneous_field_is_unchanged_by_shared_run_set() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let steady = agent("steady", (0..5).map(|_| run(0.001, 0.001, 60)).collect());
let field = [lucky, skilled, steady];
let on = rank(&field, &ScoreConfig::default());
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(on, off, "identical cells ⇒ identical board");
assert!(on
.iter()
.all(|s| s.runs_scored == 5 && s.runs_submitted == 5));
}
#[test]
fn easy_subset_entrant_is_compared_on_the_shared_cells() {
let easy = || run(0.004, 0.0005, 60);
let hard = || run(0.0, 0.004, 60);
let veteran = agent("veteran", vec![easy(), easy(), hard(), hard(), hard()]);
let entrant = agent("entrant", vec![easy(), easy()]);
let field = [veteran, entrant];
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(off[0].agent_id, "entrant");
assert!(off[0].rank_eligible && !off[1].rank_eligible);
let on = rank(&field, &ScoreConfig::default());
let get = |id: &str| on.iter().find(|s| s.agent_id == id).unwrap();
let (v, e) = (get("veteran"), get("entrant"));
assert_eq!(v.runs_scored, 2);
assert_eq!(e.runs_scored, 2);
assert_eq!(v.runs_submitted, 5);
assert_eq!(e.runs_submitted, 2);
assert_eq!(v.deflated_sharpe.to_bits(), e.deflated_sharpe.to_bits());
assert_eq!(v.rank_eligible, e.rank_eligible);
assert_eq!(v.tie_group, e.tie_group, "indistinguishable ⇒ one band");
assert!(v.dsr_tied && e.dsr_tied);
assert!(
e.rank_ordinal == 0 || v.rank_ordinal <= e.rank_ordinal,
"the entrant must not rank above the veteran"
);
}
#[test]
fn empty_submission_does_not_empty_the_shared_cells() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let ghost = agent("ghost", Vec::new());
let board = rank(&[ghost, skilled], &ScoreConfig::default());
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(get("skilled").runs_scored, 5);
assert!(get("skilled").rank_eligible);
assert!(!get("ghost").rank_eligible);
}
#[test]
fn small_field_keeps_the_configured_sr_std_byte_identical() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..4)
.map(|i| {
let m = 0.001 + 0.001 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.001, 60)).collect(),
)
})
.collect();
let board = rank(&field, &cfg);
assert_eq!(board.len(), 4);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), per_period_sr_std(&cfg).to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(cfg.trials_sr_std));
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
assert_eq!(s.deflated_sharpe.to_bits(), alone.deflated_sharpe.to_bits());
assert_eq!(s.dsr_ci_low.to_bits(), alone.dsr_ci_low.to_bits());
}
let pinned = rank(
&field,
&ScoreConfig {
min_field_for_measured_sr_std: usize::MAX,
..cfg
},
);
assert_eq!(board, pinned);
}
#[test]
fn large_field_measures_trials_sr_std_from_the_field() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.003, 60)).collect(),
)
})
.collect();
let board = rank(&field, &cfg);
let mut sharpes: Vec<f64> = field
.iter()
.map(|a| {
let pooled: Vec<f64> = a
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
sharpe_ratio(&pooled)
})
.collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let expected = std_dev(&sharpes);
assert!(expected > 0.0 && expected.is_finite());
let mut moved = false;
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
moved |= s.deflated_sharpe.to_bits() != alone.deflated_sharpe.to_bits();
}
assert!(
moved,
"measured dispersion must actually move the deflation"
);
let mut reversed = field.clone();
reversed.reverse();
let again = rank(&reversed, &cfg);
assert_eq!(again[0].trials_sr_std.to_bits(), expected.to_bits());
}
fn gaussian_like(n: usize, mean_ret: f64, sd: f64, seed: u64) -> Vec<f64> {
let mut state = seed;
let mut next = || {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
(state >> 11) as f64 / (1u64 << 53) as f64
};
let raw: Vec<f64> = (0..n)
.map(|_| (0..12).map(|_| next()).sum::<f64>() - 6.0)
.collect();
let (m, s) = (mean(&raw), std_dev(&raw));
raw.iter().map(|x| mean_ret + sd * (x - m) / s).collect()
}
fn pooled_of(sub: &AgentSubmission) -> Vec<f64> {
sub.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
}
#[test]
fn annualized_prior_is_converted_per_period_once() {
let cfg = ScoreConfig {
trials_sr_std: 0.5,
periods_per_year: 8760.0,
..ScoreConfig::default()
};
let expected = 0.5 / 8760f64.sqrt();
assert_eq!(per_period_sr_std(&cfg).to_bits(), expected.to_bits());
let sub = agent("a", (0..5).map(|_| run(0.0002, 0.003, 300)).collect());
let s = score_agent(&sub, &cfg);
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(0.5));
let pooled = pooled_of(&sub);
let once = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected);
let raw = deflated_sharpe_ratio(&pooled, cfg.n_trials, 0.5);
let twice = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected / 8760f64.sqrt());
assert_eq!(s.deflated_sharpe.to_bits(), once.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), raw.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), twice.to_bits());
}
#[test]
fn measured_sr_std_is_never_reconverted() {
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.003, 60)).collect(),
)
})
.collect();
let mut sharpes: Vec<f64> = field.iter().map(|a| sharpe_ratio(&pooled_of(a))).collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let raw_measured = std_dev(&sharpes);
for ppy in [1.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
let board = rank(&field, &cfg);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), raw_measured.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let pooled = pooled_of(field.iter().find(|a| a.agent_id == s.agent_id).unwrap());
let expected = deflated_sharpe_ratio(&pooled, cfg.n_trials, raw_measured);
assert_eq!(s.deflated_sharpe.to_bits(), expected.to_bits());
}
}
}
#[test]
fn daily_buy_and_hold_clears_the_corrected_bar() {
let runs: Vec<Run> = (0..3)
.map(|i| Run {
returns: gaussian_like(2500, 0.0004, 0.011, 0x5B_A7 + i),
..Run::default()
})
.collect();
let index = agent("buy-and-hold", runs);
let cfg = ScoreConfig {
n_trials: 50,
trials_sr_std: 0.1,
periods_per_year: 252.0,
..ScoreConfig::default()
};
let s = score_agent(&index, &cfg);
assert!(s.psr > 0.99, "the index is clearly positive: {s:?}");
assert!(
s.rank_eligible,
"an index-like daily track must clear an annualized 0.1 prior: {s:?}"
);
let old_units = ScoreConfig {
periods_per_year: 1.0,
..cfg
};
let old = score_agent(&index, &old_units);
assert!(
!old.rank_eligible && old.deflated_sharpe < 0.05,
"applied per period the same prior was unreachable: {old:?}"
);
}
#[test]
fn hourly_bar_is_stricter_per_period_than_daily_for_the_same_annualized_prior() {
let daily = ScoreConfig::for_periods_per_year(252.0);
let hourly = ScoreConfig::for_periods_per_year(8760.0);
let star = |cfg: &ScoreConfig| expected_max_sharpe(per_period_sr_std(cfg), cfg.n_trials);
assert!(star(&hourly) > 0.0 && star(&daily) > 0.0);
assert!(
star(&hourly) < star(&daily),
"hourly sr_star {} must be below daily {}",
star(&hourly),
star(&daily)
);
let ann_h = star(&hourly) * 8760f64.sqrt();
let ann_d = star(&daily) * 252f64.sqrt();
assert!((ann_h - ann_d).abs() < 1e-12, "{ann_h} vs {ann_d}");
}
#[test]
fn default_min_annual_sharpe_is_identical_to_the_old_per_run_test() {
let mut runs = vec![run(0.02, 0.002, 60), run(0.0, 0.003, 60)];
runs.extend((0..3).map(|_| run(0.002, 0.0005, 60)));
let sub = agent("mixed", runs);
let clean = agent("clean", (0..3).map(|_| run(0.002, 0.0005, 60)).collect());
for ppy in [52.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
assert_eq!(per_run_psr_benchmark(&cfg).to_bits(), 0f64.to_bits());
let old: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, 0.0) >= cfg.per_run_psr_bar)
.collect();
assert!(old.iter().any(|&p| p) && old.iter().any(|&p| !p));
assert_eq!(
score_agent(&sub, &cfg).passed_k,
pass_k(&old, PassMode::All)
);
assert!(score_agent(&clean, &cfg).passed_k);
}
let strict = ScoreConfig {
per_run_min_annual_sharpe: 3.0,
..ScoreConfig::for_periods_per_year(252.0)
};
assert_eq!(
per_run_psr_benchmark(&strict).to_bits(),
(3.0 / 252f64.sqrt()).to_bits()
);
let weak = agent("weak", (0..3).map(|_| run(0.0005, 0.004, 60)).collect());
assert!(score_agent(&weak, &ScoreConfig::for_periods_per_year(252.0)).passed_k);
assert!(
!score_agent(&weak, &strict).passed_k,
"a minimum annualized Sharpe of 3.0 must fail a 0.0005/0.004 track"
);
}
fn field_with_one_single_run_failure() -> Vec<AgentSubmission> {
let clean = |id: &str| agent(id, (0..6).map(|_| run(0.002, 0.0005, 60)).collect());
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs.push(run(-0.002, 0.0005, 60));
vec![clean("a"), agent("one_bad_run", runs), clean("b")]
}
#[test]
fn default_pass_mode_is_all_and_matches_the_old_gate() {
let cfg = ScoreConfig::default();
assert_eq!(cfg.pass_mode, PassMode::All);
assert_eq!(cfg.mandate.max_run_drawdown, 1.0);
let field = field_with_one_single_run_failure();
let board = rank(&field, &cfg);
let benchmark = per_run_psr_benchmark(&cfg);
for s in &board {
let sub = field.iter().find(|a| a.agent_id == s.agent_id).unwrap();
let old_per_run: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, benchmark) >= cfg.per_run_psr_bar)
.collect();
let old_passed_k = pass_k(&old_per_run, PassMode::All);
assert_eq!(s.passed_k, old_passed_k, "{}", s.agent_id);
let old_eligible = s.deflated_sharpe >= cfg.dsr_bar
&& old_passed_k
&& s.process_ok
&& s.bootstrap_p < cfg.alpha
&& s.max_drawdown <= cfg.mandate.max_drawdown;
assert_eq!(s.rank_eligible, old_eligible, "{}", s.agent_id);
}
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert!(get("a").rank_eligible && get("b").rank_eligible);
assert!(!get("one_bad_run").rank_eligible);
assert!(!get("one_bad_run").passed_k);
}
#[test]
fn any_mode_admits_a_regime_dependent_edge_the_all_mode_rejects() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.003, 0.0005, 60)).collect();
runs.push(run(-0.001, 0.0005, 60));
let sub = agent("regime_edge", runs);
let all = score_agent(&sub, &ScoreConfig::default());
assert!(!all.passed_k && !all.rank_eligible, "{all:?}");
let any = score_agent(&sub, &ScoreConfig::reliability_never_catastrophic(0.20));
assert!(any.deflated_sharpe >= 0.95);
assert!(any.bootstrap_p < 0.05 && any.process_ok && any.mandate_ok);
assert!(any.worst_run_drawdown < 0.20 && any.max_drawdown < 0.20);
assert!(any.passed_k && any.rank_eligible, "{any:?}");
}
#[test]
fn per_run_drawdown_bound_rejects_one_catastrophic_run_that_the_pooled_bound_misses() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
for r in &mut crash.returns[10..15] {
*r = -0.03;
}
runs.push(crash);
let sub = agent("one_blowup", runs);
let loose_pooled = ScoreConfig {
mandate: Mandate {
max_drawdown: 0.20,
max_run_drawdown: 1.0,
},
pass_mode: PassMode::Any,
..ScoreConfig::default()
};
let pooled_only = score_agent(&sub, &loose_pooled);
assert!(pooled_only.max_drawdown <= 0.20, "{pooled_only:?}");
assert!(
pooled_only.mandate_ok && pooled_only.rank_eligible,
"{pooled_only:?}"
);
let mut preset = ScoreConfig::reliability_never_catastrophic(0.10);
preset.mandate.max_drawdown = 0.20;
let bounded = score_agent(&sub, &preset);
assert!(bounded.worst_run_drawdown > 0.10, "{bounded:?}");
assert!(bounded.max_drawdown <= 0.20, "the pooled bound still holds");
assert!(bounded.passed_k, "pass^k is not what rejects it");
assert!(!bounded.mandate_ok && !bounded.rank_eligible, "{bounded:?}");
}
#[test]
fn never_catastrophic_preset_is_weaker_than_the_default() {
let mut field = field_with_one_single_run_failure();
field.pop();
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
crash.returns[10] = -0.30;
runs.push(crash);
field.push(agent("blowup", runs));
let mut lucky = vec![run(0.02, 0.002, 60)];
lucky.extend((0..5).map(|_| run(0.0, 0.003, 60)));
field.push(agent("lucky", lucky));
assert!(field.iter().all(|a| a.runs.len() == 6));
assert!(field.len() < ScoreConfig::default().min_field_for_measured_sr_std);
let default = rank(&field, &ScoreConfig::default());
let preset = rank(&field, &ScoreConfig::reliability_never_catastrophic(0.20));
let eligible = |board: &[CompositeScore]| -> Vec<String> {
let mut v: Vec<String> = board
.iter()
.filter(|s| s.rank_eligible)
.map(|s| s.agent_id.clone())
.collect();
v.sort();
v
};
let (d, p) = (eligible(&default), eligible(&preset));
assert_eq!(d, vec!["a".to_string()], "{default:?}");
assert!(
d.iter().all(|id| p.contains(id)),
"default {d:?} not within preset {p:?}"
);
assert!(p.contains(&"one_bad_run".to_string()) && !d.contains(&"one_bad_run".to_string()));
assert!(
!p.contains(&"blowup".to_string()),
"a blow-up is refused under both"
);
assert!(
p.contains(&"lucky".to_string()) && !d.contains(&"lucky".to_string()),
"{preset:?}"
);
}
#[test]
fn worst_run_drawdown_is_the_max_over_runs_not_the_pooled_track() {
let down = || Run {
returns: vec![-0.05; 4],
..Run::default()
};
let up = Run {
returns: vec![0.10; 10],
..Run::default()
};
let sub = agent("runs", vec![up.clone(), down(), down()]);
let s = score_agent(&sub, &ScoreConfig::default());
let expected = max_drawdown(&up.returns).max(max_drawdown(&down().returns));
assert_eq!(s.worst_run_drawdown.to_bits(), expected.to_bits());
assert!((s.worst_run_drawdown - (1.0 - 0.95f64.powi(4))).abs() < 1e-12);
assert_eq!(
s.max_drawdown.to_bits(),
max_drawdown(&pooled_of(&sub)).to_bits()
);
assert!((s.max_drawdown - (1.0 - 0.95f64.powi(8))).abs() < 1e-12);
assert!(s.max_drawdown > s.worst_run_drawdown);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert_eq!(empty.worst_run_drawdown, 0.0);
}
}