use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
pub use sharpebench_protocol::DeclaredMandate;
use crate::calibration::brier_score;
use crate::comparison_sets::{comparison_set, restrict_to_shared, TaggedRun, TaggedSubmission};
use crate::decay::edge_half_life;
use crate::deflated_sharpe::{
deflated_sharpe_ratio_against_null, expected_max_sharpe, probabilistic_sharpe_ratio,
sharpe_ratio,
};
use crate::econrationality::{elicit_revealed_selection, rationality_score};
use crate::pass_k::{pass_k, PassMode};
use crate::percentile::percentile_of;
use crate::process::{process_score, ProcessEvent, ProcessScore, Trace};
use crate::rediscovery::{clone_clusters, CLONE_COLLAPSE_COSINE};
use crate::roles::{attribute_behavior_roles, RoleContribution};
use crate::rolling::rolling_sharpe;
use crate::selection::{selection_robustness, SelectionRobustness};
use crate::significance::bootstrap_pvalue;
use crate::stats::{mean, std_dev};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Run {
pub returns: Vec<f64>,
#[serde(default)]
pub trace: Trace,
#[serde(default)]
pub confidences: Vec<f64>,
#[serde(default)]
pub outcomes: Vec<bool>,
#[serde(default)]
pub cost: f64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct AgentSubmission {
pub agent_id: String,
pub runs: Vec<Run>,
#[serde(default)]
pub in_sample_trials: u32,
#[serde(default)]
pub candidates: Vec<Vec<f64>>,
}
pub type MandateDeclarations = BTreeMap<String, DeclaredMandate>;
#[derive(Clone, Debug, Deserialize)]
pub struct DeclaredSubmission {
#[serde(flatten)]
pub submission: AgentSubmission,
#[serde(default)]
pub declared_mandate: Option<DeclaredMandate>,
}
pub fn split_declarations(
field: Vec<DeclaredSubmission>,
) -> (Vec<AgentSubmission>, MandateDeclarations) {
let mut declarations = MandateDeclarations::new();
let subs = field
.into_iter()
.map(|d| {
if let Some(m) = d.declared_mandate {
declarations.insert(d.submission.agent_id.clone(), m);
}
d.submission
})
.collect();
(subs, declarations)
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum MandateVerdict {
AbsoluteReturn,
RelativeTo { benchmark_id: String },
DrawdownCapped { max_per_run_drawdown: f64 },
}
impl MandateVerdict {
pub fn of(declared: &DeclaredMandate) -> Self {
match declared {
DeclaredMandate::AbsoluteReturn => Self::AbsoluteReturn,
DeclaredMandate::RelativeTo { benchmark_id } => Self::RelativeTo {
benchmark_id: benchmark_id.clone(),
},
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown,
} => Self::DrawdownCapped {
max_per_run_drawdown: *max_per_run_drawdown,
},
DeclaredMandate::OutperformBuyAndHold => Self::RelativeTo {
benchmark_id: default_benchmark_agent_id(),
},
}
}
fn pass_mode(&self) -> PassMode {
match self {
Self::AbsoluteReturn => PassMode::All,
Self::RelativeTo { .. } => PassMode::RelativeToBenchmark,
Self::DrawdownCapped { .. } => PassMode::Any,
}
}
pub fn benchmark_id(&self) -> Option<&str> {
match self {
Self::RelativeTo { benchmark_id } => Some(benchmark_id),
Self::AbsoluteReturn | Self::DrawdownCapped { .. } => None,
}
}
fn drawdown_bound_holds(&self, worst_run_drawdown: f64) -> bool {
match self {
Self::DrawdownCapped {
max_per_run_drawdown: x,
} => *x > 0.0 && *x <= 1.0 && worst_run_drawdown <= *x,
Self::AbsoluteReturn | Self::RelativeTo { .. } => true,
}
}
pub fn describe(&self) -> String {
match self {
Self::AbsoluteReturn => "absolute return".to_string(),
Self::RelativeTo { benchmark_id } => format!("relative to {benchmark_id}"),
Self::DrawdownCapped {
max_per_run_drawdown,
} => format!("drawdown capped at {max_per_run_drawdown:.2} per run"),
}
}
}
fn class_key(v: &MandateVerdict) -> (u8, &str, u64) {
match v {
MandateVerdict::AbsoluteReturn => (0, "", 0),
MandateVerdict::RelativeTo { benchmark_id } => (1, benchmark_id, 0),
MandateVerdict::DrawdownCapped {
max_per_run_drawdown,
} => (2, "", max_per_run_drawdown.to_bits()),
}
}
struct ResolvedDeclaration<'a> {
mandate: &'a DeclaredMandate,
benchmark: Option<&'a AgentSubmission>,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub enum RankKey {
#[default]
DeflatedSharpe,
Alpha,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Mandate {
pub max_drawdown: f64,
#[serde(default = "default_max_run_drawdown")]
pub max_run_drawdown: f64,
}
impl Default for Mandate {
fn default() -> Self {
Self {
max_drawdown: 1.0,
max_run_drawdown: default_max_run_drawdown(),
}
}
}
fn default_max_run_drawdown() -> f64 {
1.0
}
fn max_drawdown(returns: &[f64]) -> f64 {
let mut nav = 1.0;
let mut peak = 1.0;
let mut mdd = 0.0;
for &r in returns {
nav *= 1.0 + r;
if nav > peak {
peak = nav;
}
if peak > 0.0 {
let dd = 1.0 - nav / peak;
if dd > mdd {
mdd = dd;
}
}
}
mdd
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreConfig {
pub n_trials: u32,
pub trials_sr_std: f64,
#[serde(default)]
pub deflation_null_mean_per_period: f64,
pub dsr_bar: f64,
pub per_run_psr_bar: f64,
#[serde(default)]
pub per_run_min_annual_sharpe: f64,
#[serde(default)]
pub pass_mode: PassMode,
#[serde(default = "default_benchmark_agent_id")]
pub benchmark_agent_id: String,
#[serde(default = "default_periods_per_year")]
pub periods_per_year: f64,
pub alpha: f64,
pub bootstrap_seed: u64,
pub n_boot: usize,
pub block_prob: f64,
#[serde(default)]
pub mandate: Mandate,
#[serde(default)]
pub rank_key: RankKey,
#[serde(default)]
pub reference_dsr_population: Vec<f64>,
#[serde(default = "default_rolling_window")]
pub rolling_window: usize,
#[serde(default = "default_dsr_ci_level")]
pub dsr_ci_level: f64,
#[serde(default = "default_shared_run_set")]
pub shared_run_set: bool,
#[serde(default = "default_min_field_for_measured_sr_std")]
pub min_field_for_measured_sr_std: usize,
#[serde(default = "default_dedup_clones_for_measured_sr_std")]
pub dedup_clones_for_measured_sr_std: bool,
#[serde(default = "default_min_measured_trials_sr_std")]
pub min_measured_trials_sr_std: f64,
#[serde(default = "default_execution_seeds_per_window")]
pub execution_seeds_per_window: usize,
}
fn default_rolling_window() -> usize {
21
}
fn default_dsr_ci_level() -> f64 {
0.90
}
fn default_shared_run_set() -> bool {
true
}
fn default_min_field_for_measured_sr_std() -> usize {
5
}
fn default_dedup_clones_for_measured_sr_std() -> bool {
true
}
fn default_min_measured_trials_sr_std() -> f64 {
0.5
}
fn default_execution_seeds_per_window() -> usize {
1
}
fn default_periods_per_year() -> f64 {
252.0
}
fn default_benchmark_agent_id() -> String {
"buy-and-hold".to_string()
}
pub fn per_run_passes(
sub: &AgentSubmission,
benchmark: Option<&AgentSubmission>,
cfg: &ScoreConfig,
) -> Vec<bool> {
let bar = per_run_psr_benchmark(cfg);
match cfg.pass_mode {
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, bar) >= cfg.per_run_psr_bar)
.collect(),
PassMode::RelativeToBenchmark => sub
.runs
.iter()
.enumerate()
.map(|(i, r)| {
benchmark
.and_then(|b| b.runs.get(i))
.and_then(|b| excess_returns(&r.returns, &b.returns))
.is_some_and(|e| {
std_dev(&e) > 0.0
&& probabilistic_sharpe_ratio(&e, bar) >= cfg.per_run_psr_bar
})
})
.collect(),
}
}
fn excess_returns(returns: &[f64], benchmark: &[f64]) -> Option<Vec<f64>> {
(returns.len() == benchmark.len())
.then(|| returns.iter().zip(benchmark).map(|(a, b)| a - b).collect())
}
pub fn per_period_sr_std(cfg: &ScoreConfig) -> f64 {
cfg.trials_sr_std / cfg.periods_per_year.sqrt()
}
pub fn per_run_psr_benchmark(cfg: &ScoreConfig) -> f64 {
cfg.per_run_min_annual_sharpe / cfg.periods_per_year.sqrt()
}
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TrialsSrStdSource {
#[default]
Configured,
Measured,
MeasuredFloored,
}
impl Default for ScoreConfig {
fn default() -> Self {
Self {
n_trials: 50,
trials_sr_std: 0.5,
deflation_null_mean_per_period: 0.0,
dsr_bar: 0.95,
per_run_psr_bar: 0.90,
per_run_min_annual_sharpe: 0.0,
pass_mode: PassMode::default(),
benchmark_agent_id: default_benchmark_agent_id(),
periods_per_year: default_periods_per_year(),
alpha: 0.05,
bootstrap_seed: 0x5BA7_2026,
n_boot: 2000,
block_prob: 0.1,
mandate: Mandate::default(),
rank_key: RankKey::default(),
reference_dsr_population: Vec::new(),
rolling_window: default_rolling_window(),
dsr_ci_level: default_dsr_ci_level(),
shared_run_set: default_shared_run_set(),
min_field_for_measured_sr_std: default_min_field_for_measured_sr_std(),
dedup_clones_for_measured_sr_std: default_dedup_clones_for_measured_sr_std(),
min_measured_trials_sr_std: default_min_measured_trials_sr_std(),
execution_seeds_per_window: default_execution_seeds_per_window(),
}
}
}
impl ScoreConfig {
pub fn for_periods_per_year(periods_per_year: f64) -> Self {
Self {
periods_per_year,
..Self::default()
}
}
pub fn reliability_never_catastrophic(max_run_dd: f64) -> Self {
Self {
pass_mode: PassMode::Any,
mandate: Mandate {
max_run_drawdown: max_run_dd,
..Mandate::default()
},
..Self::default()
}
}
pub fn relative_to_benchmark(benchmark_agent_id: &str) -> Self {
Self {
pass_mode: PassMode::RelativeToBenchmark,
benchmark_agent_id: benchmark_agent_id.to_string(),
..Self::default()
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct CompositeScore {
pub agent_id: String,
pub deflated_sharpe: f64,
pub psr: f64,
pub passed_k: bool,
pub process_ok: bool,
pub bootstrap_p: f64,
pub raw_mean_return: f64,
pub rank_eligible: bool,
pub composite: f64,
pub alpha: f64,
pub beta: f64,
pub calibration_brier: Option<f64>,
pub edge_half_life: Option<f64>,
pub field_reality_check_p: f64,
pub max_drawdown: f64,
pub mandate_ok: bool,
#[serde(default)]
pub worst_run_drawdown: f64,
pub turnover: f64,
pub pareto_optimal: bool,
pub step_down_significant: bool,
pub confidence_weighted_return: f64,
pub cost: f64,
pub return_per_cost: Option<f64>,
pub field_spa_p: f64,
pub field_spa_consistent_p: f64,
#[serde(default)]
pub field_significance_benchmark: String,
pub field_crowdedness: Option<f64>,
pub in_sample_trials: u32,
pub effective_n_trials: u32,
pub dsr_percentile: Option<f64>,
pub selection_median_dsr: Option<f64>,
pub selection_gap: Option<f64>,
pub rank_ordinal: usize,
pub rolling_min_sharpe: Option<f64>,
pub rolling_frac_positive: Option<f64>,
pub sortino: Option<f64>,
pub downside_deviation: f64,
pub dsr_per_cost: Option<f64>,
pub process_floored: bool,
pub realized_floored_return: f64,
pub dsr_ci_low: f64,
pub dsr_ci_high: f64,
pub dsr_se: f64,
pub tie_group: usize,
pub dsr_tied: bool,
#[serde(default)]
pub trials_sr_std: f64,
#[serde(default)]
pub trials_sr_std_annualized: Option<f64>,
#[serde(default)]
pub trials_sr_std_annualized_equivalent: f64,
#[serde(default)]
pub deflation_bar_per_period: f64,
#[serde(default)]
pub deflation_bar_annualized_equivalent: f64,
#[serde(default)]
pub deflation_null_mean_per_period: f64,
#[serde(default)]
pub pooled_observations: usize,
#[serde(default)]
pub trials_sr_std_source: TrialsSrStdSource,
#[serde(default)]
pub runs_submitted: usize,
#[serde(default)]
pub runs_scored: usize,
#[serde(default = "default_process_score")]
pub process_score: f64,
#[serde(default)]
pub process_warnings: usize,
#[serde(default)]
pub econ_rationality_score: Option<f64>,
#[serde(default)]
pub econ_dominance_violations: Option<usize>,
#[serde(default)]
pub role_contributions: Vec<RoleContribution>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate: Option<DeclaredMandate>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub verdict_applied: Option<MandateVerdict>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_passed_k: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate_eligible: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate_ordinal: Option<usize>,
}
impl CompositeScore {
pub fn mandate_verdict_label(&self) -> Option<String> {
let verdict = self.verdict_applied.as_ref()?;
let eligibility = if self.declared_mandate_eligible == Some(true) {
"eligible"
} else {
"ineligible"
};
let host = if self.rank_eligible {
"host-board eligible"
} else {
"host-board ineligible"
};
Some(format!(
"{eligibility} under declared verdict ({}); {host}",
verdict.describe()
))
}
}
fn default_process_score() -> f64 {
1.0
}
fn dominates(a: &CompositeScore, b: &CompositeScore) -> bool {
a.raw_mean_return >= b.raw_mean_return
&& a.max_drawdown <= b.max_drawdown
&& a.turnover <= b.turnover
&& (a.raw_mean_return > b.raw_mean_return
|| a.max_drawdown < b.max_drawdown
|| a.turnover < b.turnover)
}
#[derive(Clone, Copy)]
struct Deflation {
sr_std: f64,
annualized: Option<f64>,
source: TrialsSrStdSource,
null_mean_per_period: f64,
}
impl Deflation {
fn configured(cfg: &ScoreConfig) -> Self {
Self {
sr_std: per_period_sr_std(cfg),
annualized: Some(cfg.trials_sr_std),
source: TrialsSrStdSource::Configured,
null_mean_per_period: cfg.deflation_null_mean_per_period,
}
}
fn measured(sr_std: f64, cfg: &ScoreConfig) -> Self {
let floor = cfg.min_measured_trials_sr_std / cfg.periods_per_year.sqrt();
let floored = sr_std < floor;
Self {
sr_std: sr_std.max(floor),
annualized: None,
source: if floored {
TrialsSrStdSource::MeasuredFloored
} else {
TrialsSrStdSource::Measured
},
null_mean_per_period: cfg.deflation_null_mean_per_period,
}
}
}
pub fn score_agent(sub: &AgentSubmission, cfg: &ScoreConfig) -> CompositeScore {
score_agent_with(sub, cfg, Deflation::configured(cfg), None, None)
}
pub fn score_agent_declared(
sub: &AgentSubmission,
declared: Option<&DeclaredMandate>,
cfg: &ScoreConfig,
) -> CompositeScore {
if let Some(mandate) = declared {
let verdict = MandateVerdict::of(mandate);
if verdict.benchmark_id().is_some() {
let mut score = score_agent(sub, cfg);
score.declared_mandate = Some(mandate.clone());
score.verdict_applied = Some(verdict);
return score;
}
}
let resolved = declared.map(|mandate| ResolvedDeclaration {
mandate,
benchmark: None,
});
score_agent_with(sub, cfg, Deflation::configured(cfg), None, resolved)
}
fn score_agent_with(
sub: &AgentSubmission,
cfg: &ScoreConfig,
defl: Deflation,
benchmark: Option<&AgentSubmission>,
declared: Option<ResolvedDeclaration<'_>>,
) -> CompositeScore {
let pooled = pooled_returns(sub, cfg.execution_seeds_per_window);
let psr = probabilistic_sharpe_ratio(&pooled, 0.0);
let effective_n_trials = cfg.n_trials.saturating_add(sub.in_sample_trials);
let dsr = deflated_sharpe_ratio_against_null(
&pooled,
effective_n_trials,
defl.null_mean_per_period,
defl.sr_std,
);
let deflation_bar_per_period =
defl.null_mean_per_period + expected_max_sharpe(defl.sr_std, effective_n_trials);
let per_run = per_run_passes(sub, benchmark, cfg);
let passed_k = pass_k(&per_run, cfg.pass_mode);
let per_run_process: Vec<ProcessScore> =
sub.runs.iter().map(|r| process_score(&r.trace)).collect();
let process_ok = per_run_process.iter().all(ProcessScore::is_clean);
let process_warnings: usize = per_run_process.iter().map(|p| p.warn_violations).sum();
let graded_process_score = if process_ok {
(1.0 - process_warnings as f64 * 0.1).max(0.0)
} else {
0.0
};
let bootstrap_p = bootstrap_pvalue(&pooled, cfg.bootstrap_seed, cfg.n_boot, cfg.block_prob);
let raw_mean_return = mean(&pooled);
let conf: Vec<f64> = sub
.runs
.iter()
.flat_map(|r| r.confidences.iter().copied())
.collect();
let outc: Vec<bool> = sub
.runs
.iter()
.flat_map(|r| r.outcomes.iter().copied())
.collect();
let calibration_brier = if !conf.is_empty() && !outc.is_empty() {
Some(brier_score(&conf, &outc))
} else {
None
};
let per_run_edge: Vec<f64> = sub.runs.iter().map(|r| mean(&r.returns)).collect();
let edge_half_life_periods = edge_half_life(&per_run_edge);
let mdd = max_drawdown(&pooled);
let worst_run_drawdown = sub
.runs
.iter()
.map(|r| max_drawdown(&r.returns))
.fold(0.0, f64::max);
let mandate_ok =
mdd <= cfg.mandate.max_drawdown && worst_run_drawdown <= cfg.mandate.max_run_drawdown;
let total_orders: usize = sub
.runs
.iter()
.map(|r| {
r.trace
.events
.iter()
.filter(|e| matches!(e, ProcessEvent::OrderPlaced { .. }))
.count()
})
.sum();
let turnover = total_orders as f64 / sub.runs.len().max(1) as f64;
let mut cw_num = 0.0;
let mut cw_den = 0.0;
for r in &sub.runs {
let w = if r.confidences.is_empty() {
1.0
} else {
mean(&r.confidences)
};
cw_num += w * mean(&r.returns);
cw_den += w;
}
let confidence_weighted_return = if cw_den > 0.0 {
cw_num / cw_den
} else {
raw_mean_return
};
let cost: f64 = sub.runs.iter().map(|r| r.cost).sum();
let return_per_cost = if cost > 0.0 {
Some(raw_mean_return / cost)
} else {
None
};
let dsr_percentile = if cfg.reference_dsr_population.is_empty() {
None
} else {
Some(percentile_of(dsr, &cfg.reference_dsr_population))
};
let (selection_median_dsr, selection_gap) = if sub.candidates.is_empty() {
(None, None)
} else {
let sr: SelectionRobustness =
selection_robustness(&sub.candidates, effective_n_trials, defl.sr_std);
(Some(sr.median_dsr), Some(sr.selection_gap))
};
let rolling = rolling_sharpe(&pooled, cfg.rolling_window);
let rolling_min_sharpe = rolling.map(|r| r.min_sharpe);
let rolling_frac_positive = rolling.map(|r| r.frac_positive);
let sortino = crate::stats::sortino_ratio(&pooled, 0.0);
let downside_deviation = crate::stats::downside_deviation(&pooled, 0.0);
let dsr_per_cost = if cost > 0.0 { Some(dsr / cost) } else { None };
let process_floored = !process_ok;
let realized_floored_return = if process_floored {
0.0
} else {
raw_mean_return
};
let dsr_ci = crate::significance::bootstrap_dsr_ci_against_null(
&pooled,
effective_n_trials,
defl.null_mean_per_period,
defl.sr_std,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.dsr_ci_level,
);
let econ_choice = elicit_revealed_selection(&sub.candidates, &pooled);
let (econ_rationality_score, econ_dominance_violations) = match econ_choice {
Some(choice) => {
let violations = usize::from(choice.is_dominated());
(Some(rationality_score(&[choice])), Some(violations))
}
None => (None, None),
};
let role_contributions = attribute_behavior_roles(&sub.runs);
let rank_eligible =
dsr >= cfg.dsr_bar && passed_k && process_ok && bootstrap_p < cfg.alpha && mandate_ok;
let composite = if rank_eligible { dsr } else { 0.0 };
let declared = declared.map(|d| {
let verdict = MandateVerdict::of(d.mandate);
let verdict_cfg = ScoreConfig {
pass_mode: verdict.pass_mode(),
benchmark_agent_id: verdict
.benchmark_id()
.map_or_else(|| cfg.benchmark_agent_id.clone(), str::to_string),
..cfg.clone()
};
let per_run = per_run_passes(sub, d.benchmark, &verdict_cfg);
let declared_passed_k = pass_k(&per_run, verdict_cfg.pass_mode);
let eligible = dsr >= cfg.dsr_bar
&& declared_passed_k
&& process_ok
&& bootstrap_p < cfg.alpha
&& mandate_ok
&& verdict.drawdown_bound_holds(worst_run_drawdown);
(d.mandate.clone(), verdict, declared_passed_k, eligible)
});
let (declared_mandate, verdict_applied, declared_passed_k, declared_mandate_eligible) =
match declared {
Some((m, v, p, e)) => (Some(m), Some(v), Some(p), Some(e)),
None => (None, None, None, None),
};
CompositeScore {
agent_id: sub.agent_id.clone(),
deflated_sharpe: dsr,
psr,
passed_k,
process_ok,
bootstrap_p,
raw_mean_return,
rank_eligible,
composite,
alpha: 0.0,
beta: 0.0,
calibration_brier,
edge_half_life: edge_half_life_periods,
field_reality_check_p: 1.0,
max_drawdown: mdd,
mandate_ok,
worst_run_drawdown,
turnover,
pareto_optimal: false,
step_down_significant: false,
confidence_weighted_return,
cost,
return_per_cost,
field_spa_p: 1.0,
field_spa_consistent_p: 1.0,
field_significance_benchmark: "unscored".to_string(),
field_crowdedness: None,
in_sample_trials: sub.in_sample_trials,
effective_n_trials,
dsr_percentile,
selection_median_dsr,
selection_gap,
rank_ordinal: 0,
rolling_min_sharpe,
rolling_frac_positive,
sortino,
downside_deviation,
dsr_per_cost,
process_floored,
realized_floored_return,
dsr_ci_low: dsr_ci.lower,
dsr_ci_high: dsr_ci.upper,
dsr_se: dsr_ci.se,
tie_group: 0,
dsr_tied: false,
trials_sr_std: defl.sr_std,
trials_sr_std_annualized: defl.annualized,
trials_sr_std_annualized_equivalent: defl.sr_std * cfg.periods_per_year.sqrt(),
deflation_bar_per_period,
deflation_bar_annualized_equivalent: deflation_bar_per_period * cfg.periods_per_year.sqrt(),
deflation_null_mean_per_period: defl.null_mean_per_period,
pooled_observations: pooled.len(),
trials_sr_std_source: defl.source,
runs_submitted: sub.runs.len(),
runs_scored: sub.runs.len(),
process_score: graded_process_score,
process_warnings,
econ_rationality_score,
econ_dominance_violations,
role_contributions,
declared_mandate,
verdict_applied,
declared_passed_k,
declared_mandate_eligible,
declared_mandate_ordinal: None,
}
}
pub fn pooled_returns(sub: &AgentSubmission, seeds_per_window: usize) -> Vec<f64> {
let width = seeds_per_window.max(1);
if width == 1 {
return sub
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
}
assert!(
sub.runs.len().is_multiple_of(width),
"{} runs cannot form complete {}-execution window blocks",
sub.runs.len(),
width
);
sub.runs
.chunks_exact(width)
.flat_map(|replicates| {
let len = replicates.first().map_or(0, |r| r.returns.len());
assert!(
replicates.iter().all(|r| r.returns.len() == len),
"execution replicates in one window must have equal return lengths"
);
(0..len).map(move |t| {
replicates.iter().map(|r| r.returns[t]).sum::<f64>() / replicates.len() as f64
})
})
.collect()
}
fn restrict_to_shared_positions(subs: &[AgentSubmission]) -> Vec<AgentSubmission> {
let tag = |i: usize| format!("{i:08}");
let tagged: Vec<TaggedSubmission> = subs
.iter()
.filter(|s| !s.runs.is_empty())
.map(|s| TaggedSubmission {
agent_id: s.agent_id.clone(),
runs: s
.runs
.iter()
.enumerate()
.map(|(i, run)| TaggedRun {
window_id: tag(i),
run: run.clone(),
})
.collect(),
in_sample_trials: s.in_sample_trials,
candidates: s.candidates.clone(),
})
.collect();
let roster: Vec<String> = tagged.iter().map(|s| s.agent_id.clone()).collect();
let set = comparison_set(&roster, &tagged);
let mut tagged_iter = tagged.iter();
subs.iter()
.map(|s| {
if s.runs.is_empty() {
return s.clone();
}
let t = tagged_iter
.next()
.expect("one tagged submission per non-empty submission");
restrict_to_shared(&set, t)
})
.collect()
}
fn measured_trials_sr_std(
pooled: &[Vec<f64>],
min_field: usize,
dedup_clones: bool,
) -> Option<f64> {
let qualifying: Vec<(&[f64], f64)> = pooled
.iter()
.filter(|p| p.len() >= 2)
.map(|p| (p.as_slice(), sharpe_ratio(p)))
.filter(|(_, sr)| sr.is_finite())
.collect();
let mut sharpes: Vec<f64> = if dedup_clones {
let streams: Vec<Vec<f64>> = qualifying.iter().map(|(p, _)| p.to_vec()).collect();
clone_clusters(&streams, CLONE_COLLAPSE_COSINE, false)
.iter()
.map(|members| {
let mut cluster: Vec<f64> = members.iter().map(|&i| qualifying[i].1).collect();
cluster.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
cluster[(cluster.len() - 1) / 2]
})
.collect()
} else {
qualifying.iter().map(|(_, sr)| *sr).collect()
};
if sharpes.len() < min_field.max(2) {
return None;
}
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
Some(std_dev(&sharpes))
}
pub fn rank(subs: &[AgentSubmission], cfg: &ScoreConfig) -> Vec<CompositeScore> {
rank_declared(subs, &MandateDeclarations::new(), cfg)
}
pub fn rank_declared(
subs: &[AgentSubmission],
declarations: &MandateDeclarations,
cfg: &ScoreConfig,
) -> Vec<CompositeScore> {
let restricted;
let field: &[AgentSubmission] = if cfg.shared_run_set {
restricted = restrict_to_shared_positions(subs);
&restricted
} else {
subs
};
let mut rank_cfg = cfg.clone();
rank_cfg.n_trials = cfg
.n_trials
.max(u32::try_from(field.len()).unwrap_or(u32::MAX));
let pooled: Vec<Vec<f64>> = field
.iter()
.map(|s| pooled_returns(s, rank_cfg.execution_seeds_per_window))
.collect();
let min_len = pooled.iter().map(Vec::len).min().unwrap_or(0);
let n_agents = pooled.len().max(1) as f64;
let market: Vec<f64> = (0..min_len)
.map(|i| pooled.iter().map(|p| p[i]).sum::<f64>() / n_agents)
.collect();
let (significance_benchmark, significance_benchmark_label): (Vec<f64>, String) = field
.iter()
.position(|s| s.agent_id == rank_cfg.benchmark_agent_id)
.filter(|&idx| pooled[idx].len() >= min_len)
.map(|idx| {
(
pooled[idx][..min_len].to_vec(),
rank_cfg.benchmark_agent_id.clone(),
)
})
.unwrap_or_else(|| (vec![0.0; min_len], "zero-return-cash".to_string()));
let defl = measured_trials_sr_std(
&pooled,
cfg.min_field_for_measured_sr_std,
cfg.dedup_clones_for_measured_sr_std,
)
.map_or_else(
|| Deflation::configured(cfg),
|sr_std| Deflation::measured(sr_std, cfg),
);
let benchmark = match cfg.pass_mode {
PassMode::RelativeToBenchmark => {
field.iter().find(|s| s.agent_id == cfg.benchmark_agent_id)
}
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => None,
};
let mut scores: Vec<CompositeScore> = field
.iter()
.enumerate()
.map(|(idx, s)| {
let declared = declarations
.get(&s.agent_id)
.map(|mandate| ResolvedDeclaration {
mandate,
benchmark: MandateVerdict::of(mandate)
.benchmark_id()
.and_then(|id| field.iter().find(|b| b.agent_id == id)),
});
let mut cs = score_agent_with(s, &rank_cfg, defl, benchmark, declared);
cs.runs_submitted = subs[idx].runs.len();
cs.field_significance_benchmark = significance_benchmark_label.clone();
if min_len >= 2 {
let (alpha, beta) = crate::attribution::alpha_beta(&pooled[idx], &market);
cs.alpha = alpha;
cs.beta = beta;
}
cs
})
.collect();
if min_len >= 2 {
let field_excess: Vec<Vec<f64>> = pooled
.iter()
.map(|p| {
p.iter()
.take(min_len)
.zip(significance_benchmark.iter())
.map(|(a, b)| a - b)
.collect()
})
.collect();
let rc_p = crate::significance::reality_check_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_p = crate::significance::spa_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_c_p = crate::significance::spa_consistent_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
for cs in scores.iter_mut() {
cs.field_reality_check_p = rc_p;
cs.field_spa_p = spa_p;
cs.field_spa_consistent_p = spa_c_p;
}
let sd = crate::significance::step_down_significant(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.alpha,
);
for (cs, s) in scores.iter_mut().zip(sd) {
cs.step_down_significant = s;
}
}
if min_len >= 2 && pooled.len() >= 2 {
let aligned: Vec<&[f64]> = pooled.iter().map(|p| &p[..min_len]).collect();
for (idx, cs) in scores.iter_mut().enumerate() {
let peers: Vec<&[f64]> = aligned
.iter()
.enumerate()
.filter(|&(j, _)| j != idx)
.map(|(_, &p)| p)
.collect();
cs.field_crowdedness = crate::correlation::crowdedness(aligned[idx], &peers).mean_corr;
}
}
let pareto: Vec<bool> = (0..scores.len())
.map(|i| !(0..scores.len()).any(|j| j != i && dominates(&scores[j], &scores[i])))
.collect();
for (cs, p) in scores.iter_mut().zip(pareto) {
cs.pareto_optimal = p;
}
let sort_key = |s: &CompositeScore| match cfg.rank_key {
RankKey::DeflatedSharpe => s.composite,
RankKey::Alpha => {
if s.rank_eligible {
s.alpha
} else {
f64::NEG_INFINITY
}
}
};
scores.sort_by(|a, b| {
b.rank_eligible
.cmp(&a.rank_eligible)
.then(
sort_key(b)
.partial_cmp(&sort_key(a))
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
b.raw_mean_return
.partial_cmp(&a.raw_mean_return)
.unwrap_or(std::cmp::Ordering::Equal),
)
});
let mut group = 0usize;
let mut prev: Option<usize> = None;
for i in 0..scores.len() {
if !scores[i].rank_eligible {
continue;
}
let same_band = matches!(prev, Some(p) if ci_overlap(&scores[p], &scores[i]));
if !same_band {
group += 1;
}
scores[i].tie_group = group;
prev = Some(i);
}
let mut i = 0;
while i < scores.len() {
if !scores[i].rank_eligible {
i += 1;
continue;
}
let band = scores[i].tie_group;
let mut j = i + 1;
while j < scores.len() && scores[j].rank_eligible && scores[j].tie_group == band {
j += 1;
}
scores[i..j].sort_by(|a, b| {
b.process_score
.partial_cmp(&a.process_score)
.unwrap_or(std::cmp::Ordering::Equal)
});
i = j;
}
let mut ord = 0usize;
for cs in scores.iter_mut() {
if cs.rank_eligible {
ord += 1;
cs.rank_ordinal = ord;
}
}
let mut band_counts = vec![0usize; group + 1];
for cs in &scores {
if cs.rank_eligible {
band_counts[cs.tie_group] += 1;
}
}
for cs in scores.iter_mut() {
if cs.rank_eligible {
cs.dsr_tied = band_counts[cs.tie_group] > 1;
}
}
let mut classes: Vec<(&MandateVerdict, usize)> = scores
.iter()
.enumerate()
.filter(|(_, s)| s.declared_mandate_eligible == Some(true))
.filter_map(|(i, s)| s.verdict_applied.as_ref().map(|v| (v, i)))
.collect();
classes.sort_by(|(va, ia), (vb, ib)| {
let (a, b) = (&scores[*ia], &scores[*ib]);
class_key(va)
.cmp(&class_key(vb))
.then(
b.deflated_sharpe
.partial_cmp(&a.deflated_sharpe)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(a.agent_id.cmp(&b.agent_id))
});
let mut ordinals: Vec<(usize, usize)> = Vec::with_capacity(classes.len());
let mut prev: Option<&MandateVerdict> = None;
let mut ord = 0usize;
for (v, i) in classes {
if prev != Some(v) {
ord = 0;
}
ord += 1;
ordinals.push((i, ord));
prev = Some(v);
}
for (i, ord) in ordinals {
scores[i].declared_mandate_ordinal = Some(ord);
}
scores
}
fn ci_overlap(a: &CompositeScore, b: &CompositeScore) -> bool {
a.dsr_ci_low <= b.dsr_ci_high && b.dsr_ci_low <= a.dsr_ci_high
}
#[cfg(test)]
mod tests {
use super::*;
use crate::deflated_sharpe::{deflated_sharpe_ratio, expected_max_sharpe};
use crate::process::ProcessEvent;
fn run(mean_ret: f64, amp: f64, n: usize) -> Run {
let returns = (0..n)
.map(|i| mean_ret + amp * (i as f64 * 0.7).sin())
.collect();
Run {
returns,
trace: Trace::default(),
confidences: Vec::new(),
outcomes: Vec::new(),
cost: 0.0,
}
}
fn agent(id: &str, runs: Vec<Run>) -> AgentSubmission {
AgentSubmission {
agent_id: id.to_string(),
runs,
in_sample_trials: 0,
candidates: Vec::new(),
}
}
#[test]
fn skilled_is_eligible() {
let s = score_agent(
&agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rank_eligible, "skilled should be eligible: {s:?}");
assert!(s.passed_k && s.process_ok);
}
#[test]
fn lucky_high_return_fails_pass_k() {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
let s = score_agent(&agent("lucky", runs), &ScoreConfig::default());
assert!(!s.passed_k, "lucky should fail pass^k");
assert!(!s.rank_eligible, "lucky must not be rank-eligible: {s:?}");
}
#[test]
fn process_violator_is_disqualified() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("violator", runs), &ScoreConfig::default());
assert!(!s.process_ok);
assert!(!s.rank_eligible, "a risk-gate bypass must disqualify");
}
#[test]
fn deflation_demotes_luck() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky.clone(), skilled.clone()], &ScoreConfig::default());
let lucky_raw = board
.iter()
.find(|s| s.agent_id == "lucky")
.unwrap()
.raw_mean_return;
let skilled_raw = board
.iter()
.find(|s| s.agent_id == "skilled")
.unwrap()
.raw_mean_return;
assert!(
lucky_raw > skilled_raw,
"lucky raw {lucky_raw} should exceed skilled {skilled_raw}"
);
assert_eq!(board[0].agent_id, "skilled");
assert!(board[0].rank_eligible && !board[1].rank_eligible);
}
#[test]
fn confidence_weighting_rewards_conviction() {
let win = Run {
returns: vec![0.01; 30],
trace: Trace::default(),
confidences: vec![0.9; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let lose = Run {
returns: vec![-0.005; 30],
trace: Trace::default(),
confidences: vec![0.1; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let s = score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default());
assert!(
s.confidence_weighted_return > s.raw_mean_return,
"cwr {} should beat raw {}",
s.confidence_weighted_return,
s.raw_mean_return
);
}
#[test]
fn cost_efficiency_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 30);
r.cost = 4.0;
let s = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
assert_eq!(s.cost, 4.0);
assert!(s.return_per_cost.is_some());
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 30)]),
&ScoreConfig::default(),
);
assert!(free.return_per_cost.is_none());
}
#[test]
fn in_sample_search_raises_the_deflation_bar() {
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let base = score_agent(&agent("base", runs.clone()), &ScoreConfig::default());
let mut over = agent("over", runs);
over.in_sample_trials = 5000;
let s = score_agent(&over, &ScoreConfig::default());
assert_eq!(s.effective_n_trials, 5050);
assert!(
s.deflated_sharpe <= base.deflated_sharpe,
"more in-sample search must not raise DSR ({} vs {})",
s.deflated_sharpe,
base.deflated_sharpe
);
}
#[test]
fn ranked_field_size_is_an_observable_trial_floor() {
let field: Vec<_> = (0..8)
.map(|i| {
agent(
&format!("agent-{i}"),
vec![run(0.002 + f64::from(i) * 0.00001, 0.0005, 60)],
)
})
.collect();
let cfg = ScoreConfig {
n_trials: 1,
min_field_for_measured_sr_std: usize::MAX,
..ScoreConfig::default()
};
let board = rank(&field, &cfg);
assert!(board.iter().all(|score| score.effective_n_trials == 8));
let standalone = score_agent(&field[0], &cfg);
assert_eq!(standalone.effective_n_trials, 1);
}
#[test]
fn percentile_reported_only_with_reference() {
let none = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(none.dsr_percentile.is_none());
let cfg = ScoreConfig {
reference_dsr_population: vec![0.0, 0.3, 0.6, 0.9],
..ScoreConfig::default()
};
let some = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&cfg,
);
assert!(some.dsr_percentile.is_some());
}
#[test]
fn rolling_sharpe_reported_for_long_tracks() {
let s = score_agent(
&agent("roll", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rolling_min_sharpe.is_some());
let fp = s.rolling_frac_positive.expect("reported");
assert!(
(fp - 1.0).abs() < 1e-12,
"steady edge → all windows positive"
);
}
#[test]
fn rolling_sharpe_none_when_track_too_short() {
let cfg = ScoreConfig {
rolling_window: 100,
..ScoreConfig::default()
};
let s = score_agent(&agent("short", vec![run(0.002, 0.0005, 30)]), &cfg);
assert!(s.rolling_min_sharpe.is_none());
assert!(s.rolling_frac_positive.is_none());
}
#[test]
fn dsr_per_cost_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 60);
r.cost = 5.0;
let paid = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
let dpc = paid.dsr_per_cost.expect("reported with cost");
assert!((dpc - paid.deflated_sharpe / 5.0).abs() < 1e-12);
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 60)]),
&ScoreConfig::default(),
);
assert!(free.dsr_per_cost.is_none());
}
#[test]
fn process_violation_floors_realized_return() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.02, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("cheater", runs), &ScoreConfig::default());
assert!(s.process_floored, "block violation must set the floor flag");
assert_eq!(
s.realized_floored_return, 0.0,
"floored to no-skill baseline"
);
assert!(
s.raw_mean_return > 0.0,
"raw return is preserved un-floored"
);
assert!(!s.rank_eligible, "eligibility logic intact");
}
#[test]
fn clean_process_is_not_floored() {
let s = score_agent(
&agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(!s.process_floored);
assert_eq!(s.realized_floored_return, s.raw_mean_return);
}
#[test]
fn overlapping_dsr_cis_flag_a_tie_and_separation_gets_a_distinct_band() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let strong_a = agent("strong_a", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let strong_b = agent("strong_b", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let weak = agent("weak", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[strong_a, strong_b, weak], &cfg);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let (a, b, w) = (get("strong_a"), get("strong_b"), get("weak"));
assert!(
a.rank_eligible && b.rank_eligible && w.rank_eligible,
"all three should clear the (deliberately low) bar"
);
assert_eq!(a.tie_group, b.tie_group, "identical CIs share a band");
assert!(a.dsr_tied && b.dsr_tied, "the pair is flagged tied");
assert_ne!(
w.tie_group, a.tie_group,
"separable agent gets a distinct band"
);
assert!(!w.dsr_tied, "a distinct band is not a tie");
assert!(
w.dsr_ci_high < a.dsr_ci_low,
"weak CI upper {} should sit below strong CI lower {}",
w.dsr_ci_high,
a.dsr_ci_low
);
}
#[test]
fn rank_ordinal_is_one_based_among_eligible() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky, skilled], &ScoreConfig::default());
assert_eq!(board[0].rank_ordinal, 1, "leader is ordinal 1");
assert_eq!(board[1].rank_ordinal, 0, "ineligible gets ordinal 0");
}
#[test]
fn homogeneous_field_is_unchanged_by_shared_run_set() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let steady = agent("steady", (0..5).map(|_| run(0.001, 0.001, 60)).collect());
let field = [lucky, skilled, steady];
let on = rank(&field, &ScoreConfig::default());
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(on, off, "identical cells ⇒ identical board");
assert!(on
.iter()
.all(|s| s.runs_scored == 5 && s.runs_submitted == 5));
}
#[test]
fn easy_subset_entrant_is_compared_on_the_shared_cells() {
let easy = || run(0.004, 0.0005, 60);
let hard = || run(0.0, 0.004, 60);
let veteran = agent("veteran", vec![easy(), easy(), hard(), hard(), hard()]);
let entrant = agent("entrant", vec![easy(), easy()]);
let field = [veteran, entrant];
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(off[0].agent_id, "entrant");
assert!(off[0].rank_eligible && !off[1].rank_eligible);
let on = rank(&field, &ScoreConfig::default());
let get = |id: &str| on.iter().find(|s| s.agent_id == id).unwrap();
let (v, e) = (get("veteran"), get("entrant"));
assert_eq!(v.runs_scored, 2);
assert_eq!(e.runs_scored, 2);
assert_eq!(v.runs_submitted, 5);
assert_eq!(e.runs_submitted, 2);
assert_eq!(v.deflated_sharpe.to_bits(), e.deflated_sharpe.to_bits());
assert_eq!(v.rank_eligible, e.rank_eligible);
assert_eq!(v.tie_group, e.tie_group, "indistinguishable ⇒ one band");
assert!(v.dsr_tied && e.dsr_tied);
assert!(
e.rank_ordinal == 0 || v.rank_ordinal <= e.rank_ordinal,
"the entrant must not rank above the veteran"
);
}
#[test]
fn empty_submission_does_not_empty_the_shared_cells() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let ghost = agent("ghost", Vec::new());
let board = rank(&[ghost, skilled], &ScoreConfig::default());
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(get("skilled").runs_scored, 5);
assert!(get("skilled").rank_eligible);
assert!(!get("ghost").rank_eligible);
}
#[test]
fn small_field_keeps_the_configured_sr_std_byte_identical() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..4)
.map(|i| {
let m = 0.001 + 0.001 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.001, 60)).collect(),
)
})
.collect();
let board = rank(&field, &cfg);
assert_eq!(board.len(), 4);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), per_period_sr_std(&cfg).to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(cfg.trials_sr_std));
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
assert_eq!(s.deflated_sharpe.to_bits(), alone.deflated_sharpe.to_bits());
assert_eq!(s.dsr_ci_low.to_bits(), alone.dsr_ci_low.to_bits());
}
let pinned = rank(
&field,
&ScoreConfig {
min_field_for_measured_sr_std: usize::MAX,
..cfg
},
);
assert_eq!(board, pinned);
}
#[test]
fn large_field_measures_trials_sr_std_from_the_field() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let board = rank(&field, &cfg);
let mut sharpes: Vec<f64> = field
.iter()
.map(|a| {
let pooled: Vec<f64> = a
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
sharpe_ratio(&pooled)
})
.collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let expected = std_dev(&sharpes);
assert!(expected > 0.0 && expected.is_finite());
let mut moved = false;
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
moved |= s.deflated_sharpe.to_bits() != alone.deflated_sharpe.to_bits();
}
assert!(
moved,
"measured dispersion must actually move the deflation"
);
let mut reversed = field.clone();
reversed.reverse();
let again = rank(&reversed, &cfg);
assert_eq!(again[0].trials_sr_std.to_bits(), expected.to_bits());
}
#[test]
fn measured_dispersion_cannot_fall_below_the_precommitted_floor() {
let cfg = ScoreConfig {
dedup_clones_for_measured_sr_std: false,
min_measured_trials_sr_std: 0.5,
..ScoreConfig::default()
};
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let phase = i as f64 * 36.0;
let returns = (0..180)
.map(|t| {
0.001 + 0.0004 * (std::f64::consts::TAU * (t as f64 + phase) / 180.0).sin()
})
.collect();
agent(
&format!("low-dispersion-{i}"),
vec![Run {
returns,
..Run::default()
}],
)
})
.collect();
let board = rank(&field, &cfg);
let floor = per_period_sr_std(&cfg);
assert!(board.iter().all(|s| {
s.trials_sr_std_source == TrialsSrStdSource::MeasuredFloored
&& s.trials_sr_std.to_bits() == floor.to_bits()
}));
}
fn gaussian_like(n: usize, mean_ret: f64, sd: f64, seed: u64) -> Vec<f64> {
let mut state = seed;
let mut next = || {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
(state >> 11) as f64 / (1u64 << 53) as f64
};
let raw: Vec<f64> = (0..n)
.map(|_| (0..12).map(|_| next()).sum::<f64>() - 6.0)
.collect();
let (m, s) = (mean(&raw), std_dev(&raw));
raw.iter().map(|x| mean_ret + sd * (x - m) / s).collect()
}
fn pooled_of(sub: &AgentSubmission) -> Vec<f64> {
sub.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
}
#[test]
fn execution_replicates_are_averaged_not_counted_as_extra_market_time() {
let sub = agent(
"replicated",
vec![
run(0.01, 0.0, 3),
run(0.03, 0.0, 3),
run(-0.02, 0.0, 3),
run(0.00, 0.0, 3),
],
);
let cfg = ScoreConfig {
execution_seeds_per_window: 2,
..ScoreConfig::default()
};
let pooled = pooled_returns(&sub, cfg.execution_seeds_per_window);
assert_eq!(pooled.len(), 6);
assert_eq!(pooled, vec![0.02, 0.02, 0.02, -0.01, -0.01, -0.01]);
let score = score_agent(&sub, &cfg);
assert_eq!(score.pooled_observations, 6);
assert_eq!(
score.psr.to_bits(),
probabilistic_sharpe_ratio(&pooled, 0.0).to_bits(),
"PSR must see the de-duplicated time axis"
);
}
#[test]
fn one_execution_per_window_preserves_legacy_pooling() {
let sub = agent("single", vec![run(0.01, 0.002, 4), run(-0.01, 0.001, 5)]);
assert_eq!(
pooled_returns(&sub, 1),
pooled_of(&sub),
"the default is byte-compatible for one execution per window"
);
}
#[test]
#[should_panic(expected = "cannot form complete")]
fn incomplete_execution_replicate_block_is_rejected() {
let sub = agent(
"bad",
vec![
run(0.01, 0.001, 3),
run(0.01, 0.001, 3),
run(0.01, 0.001, 3),
],
);
let _ = pooled_returns(&sub, 2);
}
#[test]
#[should_panic(expected = "equal return lengths")]
fn unequal_execution_replicate_lengths_are_rejected() {
let sub = agent("bad", vec![run(0.01, 0.001, 3), run(0.01, 0.001, 4)]);
let _ = pooled_returns(&sub, 2);
}
#[test]
fn annualized_prior_is_converted_per_period_once() {
let cfg = ScoreConfig {
trials_sr_std: 0.5,
periods_per_year: 8760.0,
..ScoreConfig::default()
};
let expected = 0.5 / 8760f64.sqrt();
assert_eq!(per_period_sr_std(&cfg).to_bits(), expected.to_bits());
let sub = agent("a", (0..5).map(|_| run(0.0002, 0.003, 300)).collect());
let s = score_agent(&sub, &cfg);
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(0.5));
let pooled = pooled_of(&sub);
let once = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected);
let raw = deflated_sharpe_ratio(&pooled, cfg.n_trials, 0.5);
let twice = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected / 8760f64.sqrt());
assert_eq!(s.deflated_sharpe.to_bits(), once.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), raw.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), twice.to_bits());
}
#[test]
fn clone_clusters_vote_once_on_the_measured_sr_std() {
let cfg = ScoreConfig::default();
let distinct: Vec<AgentSubmission> = (0..5)
.map(|i| {
agent(
&format!("a{i}"),
vec![Run {
returns: gaussian_like(120, 0.0002 + 0.0003 * i as f64, 0.003, 11 + i),
trace: Trace::default(),
confidences: vec![],
outcomes: vec![],
cost: 0.0,
}],
)
})
.collect();
let base = rank(&distinct, &cfg);
assert_eq!(base[0].trials_sr_std_source, TrialsSrStdSource::Measured);
let off = rank(
&distinct,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert_eq!(base, off, "no clones: the collapse is the identity");
let mut flooded = distinct.clone();
for k in 0..20 {
let mut copy = distinct[2].clone();
copy.agent_id = format!("copy{k:02}");
let lever = 1.0 + 0.05 * k as f64;
copy.runs[0].returns.iter_mut().for_each(|r| *r *= lever);
flooded.push(copy);
}
let board = rank(&flooded, &cfg);
assert_eq!(board.len(), 25, "clones are still scored");
assert_eq!(
board[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits(),
"twenty copies of one stream are one vote"
);
let mut shuffled = flooded.clone();
shuffled.rotate_left(7);
shuffled.swap(0, 12);
let again = rank(&shuffled, &cfg);
assert_eq!(
again[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits()
);
let exposed = rank(
&flooded,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert!(
exposed[0].trials_sr_std < base[0].trials_sr_std,
"without the collapse the copies shrink the dispersion"
);
}
#[test]
fn measured_sr_std_is_never_reconverted() {
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let mut sharpes: Vec<f64> = field.iter().map(|a| sharpe_ratio(&pooled_of(a))).collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let raw_measured = std_dev(&sharpes);
for ppy in [1.0, 252.0, 8760.0] {
let cfg = ScoreConfig {
min_measured_trials_sr_std: 0.0,
..ScoreConfig::for_periods_per_year(ppy)
};
let board = rank(&field, &cfg);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), raw_measured.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let pooled = pooled_of(field.iter().find(|a| a.agent_id == s.agent_id).unwrap());
let expected = deflated_sharpe_ratio(&pooled, cfg.n_trials, raw_measured);
assert_eq!(s.deflated_sharpe.to_bits(), expected.to_bits());
}
}
}
#[test]
fn daily_buy_and_hold_clears_the_corrected_bar() {
let runs: Vec<Run> = (0..3)
.map(|i| Run {
returns: gaussian_like(2500, 0.0004, 0.011, 0x5B_A7 + i),
..Run::default()
})
.collect();
let index = agent("buy-and-hold", runs);
let cfg = ScoreConfig {
n_trials: 50,
trials_sr_std: 0.1,
periods_per_year: 252.0,
..ScoreConfig::default()
};
let s = score_agent(&index, &cfg);
assert!(s.psr > 0.99, "the index is clearly positive: {s:?}");
assert!(
s.rank_eligible,
"an index-like daily track must clear an annualized 0.1 prior: {s:?}"
);
let old_units = ScoreConfig {
periods_per_year: 1.0,
..cfg
};
let old = score_agent(&index, &old_units);
assert!(
!old.rank_eligible && old.deflated_sharpe < 0.05,
"applied per period the same prior was unreachable: {old:?}"
);
}
#[test]
fn hourly_bar_is_stricter_per_period_than_daily_for_the_same_annualized_prior() {
let daily = ScoreConfig::for_periods_per_year(252.0);
let hourly = ScoreConfig::for_periods_per_year(8760.0);
let star = |cfg: &ScoreConfig| expected_max_sharpe(per_period_sr_std(cfg), cfg.n_trials);
assert!(star(&hourly) > 0.0 && star(&daily) > 0.0);
assert!(
star(&hourly) < star(&daily),
"hourly sr_star {} must be below daily {}",
star(&hourly),
star(&daily)
);
let ann_h = star(&hourly) * 8760f64.sqrt();
let ann_d = star(&daily) * 252f64.sqrt();
assert!((ann_h - ann_d).abs() < 1e-12, "{ann_h} vs {ann_d}");
}
#[test]
fn default_min_annual_sharpe_is_identical_to_the_old_per_run_test() {
let mut runs = vec![run(0.02, 0.002, 60), run(0.0, 0.003, 60)];
runs.extend((0..3).map(|_| run(0.002, 0.0005, 60)));
let sub = agent("mixed", runs);
let clean = agent("clean", (0..3).map(|_| run(0.002, 0.0005, 60)).collect());
for ppy in [52.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
assert_eq!(per_run_psr_benchmark(&cfg).to_bits(), 0f64.to_bits());
let old: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, 0.0) >= cfg.per_run_psr_bar)
.collect();
assert!(old.iter().any(|&p| p) && old.iter().any(|&p| !p));
assert_eq!(
score_agent(&sub, &cfg).passed_k,
pass_k(&old, PassMode::All)
);
assert!(score_agent(&clean, &cfg).passed_k);
}
let strict = ScoreConfig {
per_run_min_annual_sharpe: 3.0,
..ScoreConfig::for_periods_per_year(252.0)
};
assert_eq!(
per_run_psr_benchmark(&strict).to_bits(),
(3.0 / 252f64.sqrt()).to_bits()
);
let weak = agent("weak", (0..3).map(|_| run(0.0005, 0.004, 60)).collect());
assert!(score_agent(&weak, &ScoreConfig::for_periods_per_year(252.0)).passed_k);
assert!(
!score_agent(&weak, &strict).passed_k,
"a minimum annualized Sharpe of 3.0 must fail a 0.0005/0.004 track"
);
}
fn field_with_one_single_run_failure() -> Vec<AgentSubmission> {
let clean = |id: &str| agent(id, (0..6).map(|_| run(0.002, 0.0005, 60)).collect());
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs.push(run(-0.002, 0.0005, 60));
vec![clean("a"), agent("one_bad_run", runs), clean("b")]
}
#[test]
fn default_pass_mode_is_all_and_matches_the_old_gate() {
let cfg = ScoreConfig::default();
assert_eq!(cfg.pass_mode, PassMode::All);
assert_eq!(cfg.mandate.max_run_drawdown, 1.0);
let field = field_with_one_single_run_failure();
let board = rank(&field, &cfg);
let benchmark = per_run_psr_benchmark(&cfg);
for s in &board {
let sub = field.iter().find(|a| a.agent_id == s.agent_id).unwrap();
let old_per_run: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, benchmark) >= cfg.per_run_psr_bar)
.collect();
let old_passed_k = pass_k(&old_per_run, PassMode::All);
assert_eq!(s.passed_k, old_passed_k, "{}", s.agent_id);
let old_eligible = s.deflated_sharpe >= cfg.dsr_bar
&& old_passed_k
&& s.process_ok
&& s.bootstrap_p < cfg.alpha
&& s.max_drawdown <= cfg.mandate.max_drawdown;
assert_eq!(s.rank_eligible, old_eligible, "{}", s.agent_id);
}
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert!(get("a").rank_eligible && get("b").rank_eligible);
assert!(!get("one_bad_run").rank_eligible);
assert!(!get("one_bad_run").passed_k);
}
#[test]
fn any_mode_admits_a_regime_dependent_edge_the_all_mode_rejects() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.003, 0.0005, 60)).collect();
runs.push(run(-0.001, 0.0005, 60));
let sub = agent("regime_edge", runs);
let all = score_agent(&sub, &ScoreConfig::default());
assert!(!all.passed_k && !all.rank_eligible, "{all:?}");
let any = score_agent(&sub, &ScoreConfig::reliability_never_catastrophic(0.20));
assert!(any.deflated_sharpe >= 0.95);
assert!(any.bootstrap_p < 0.05 && any.process_ok && any.mandate_ok);
assert!(any.worst_run_drawdown < 0.20 && any.max_drawdown < 0.20);
assert!(any.passed_k && any.rank_eligible, "{any:?}");
}
#[test]
fn per_run_drawdown_bound_rejects_one_catastrophic_run_that_the_pooled_bound_misses() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
for r in &mut crash.returns[10..15] {
*r = -0.03;
}
runs.push(crash);
let sub = agent("one_blowup", runs);
let loose_pooled = ScoreConfig {
mandate: Mandate {
max_drawdown: 0.20,
max_run_drawdown: 1.0,
},
pass_mode: PassMode::Any,
..ScoreConfig::default()
};
let pooled_only = score_agent(&sub, &loose_pooled);
assert!(pooled_only.max_drawdown <= 0.20, "{pooled_only:?}");
assert!(
pooled_only.mandate_ok && pooled_only.rank_eligible,
"{pooled_only:?}"
);
let mut preset = ScoreConfig::reliability_never_catastrophic(0.10);
preset.mandate.max_drawdown = 0.20;
let bounded = score_agent(&sub, &preset);
assert!(bounded.worst_run_drawdown > 0.10, "{bounded:?}");
assert!(bounded.max_drawdown <= 0.20, "the pooled bound still holds");
assert!(bounded.passed_k, "pass^k is not what rejects it");
assert!(!bounded.mandate_ok && !bounded.rank_eligible, "{bounded:?}");
}
#[test]
fn never_catastrophic_preset_is_weaker_than_the_default() {
let mut field = field_with_one_single_run_failure();
field.pop();
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
crash.returns[10] = -0.30;
runs.push(crash);
field.push(agent("blowup", runs));
let mut lucky = vec![run(0.02, 0.002, 60)];
lucky.extend((0..5).map(|_| run(0.0, 0.003, 60)));
field.push(agent("lucky", lucky));
assert!(field.iter().all(|a| a.runs.len() == 6));
assert!(field.len() < ScoreConfig::default().min_field_for_measured_sr_std);
let default = rank(&field, &ScoreConfig::default());
let preset = rank(&field, &ScoreConfig::reliability_never_catastrophic(0.20));
let eligible = |board: &[CompositeScore]| -> Vec<String> {
let mut v: Vec<String> = board
.iter()
.filter(|s| s.rank_eligible)
.map(|s| s.agent_id.clone())
.collect();
v.sort();
v
};
let (d, p) = (eligible(&default), eligible(&preset));
assert_eq!(d, vec!["a".to_string()], "{default:?}");
assert!(
d.iter().all(|id| p.contains(id)),
"default {d:?} not within preset {p:?}"
);
assert!(p.contains(&"one_bad_run".to_string()) && !d.contains(&"one_bad_run".to_string()));
assert!(
!p.contains(&"blowup".to_string()),
"a blow-up is refused under both"
);
assert!(
p.contains(&"lucky".to_string()) && !d.contains(&"lucky".to_string()),
"{preset:?}"
);
}
fn relative_field() -> Vec<AgentSubmission> {
let bench: Vec<Run> = (0..5)
.map(|_| run(0.002, 0.0005, 60))
.chain(std::iter::once(run(-0.003, 0.0005, 60)))
.collect();
let beats: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for (i, x) in r.returns.iter_mut().enumerate() {
*x += 0.001 + 0.0002 * (i as f64 * 0.7).sin();
}
r
})
.collect();
let mut five_of_six = beats.clone();
for (i, x) in five_of_six[5].returns.iter_mut().enumerate() {
*x = bench[5].returns[i] - 0.001;
}
let trails: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for x in r.returns.iter_mut() {
*x = x.abs() * 0.5 + 0.0005;
}
r
})
.collect();
vec![
agent("buy-and-hold", bench.clone()),
agent("beats", beats),
agent("five-of-six", five_of_six),
agent("trails", trails),
agent("clone", bench),
]
}
#[test]
fn fixed_significance_benchmark_is_separate_from_default_pass_verdict() {
let field = relative_field();
let default = rank(&field, &ScoreConfig::default());
let renamed = rank(
&field,
&ScoreConfig {
benchmark_agent_id: "no-such-agent".to_string(),
..ScoreConfig::default()
},
);
assert_eq!(
default.iter().map(|s| s.rank_eligible).collect::<Vec<_>>(),
renamed.iter().map(|s| s.rank_eligible).collect::<Vec<_>>(),
"changing the significance null must not change default pass^k eligibility"
);
assert!(default
.iter()
.all(|s| s.field_significance_benchmark == "buy-and-hold"));
assert!(renamed
.iter()
.all(|s| s.field_significance_benchmark == "zero-return-cash"));
assert_eq!(
serde_json::from_str::<ScoreConfig>("{\"n_trials\":1,\"trials_sr_std\":0.5,\"dsr_bar\":0.9,\"per_run_psr_bar\":0.9,\"alpha\":0.05,\"bootstrap_seed\":1,\"n_boot\":10,\"block_prob\":0.1}")
.unwrap()
.benchmark_agent_id,
"buy-and-hold"
);
}
#[test]
fn relative_verdict_tests_excess_over_the_same_cell() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("buy-and-hold");
let default = rank(&field, &ScoreConfig::default());
let relative = rank(&field, &cfg);
let passed = |board: &[CompositeScore], id: &str| {
board.iter().find(|s| s.agent_id == id).unwrap().passed_k
};
assert!(!passed(&default, "beats") && passed(&relative, "beats"));
assert!(passed(&default, "trails") && !passed(&relative, "trails"));
assert!(!passed(&default, "buy-and-hold") && !passed(&relative, "buy-and-hold"));
assert!(!passed(&relative, "clone"));
assert!(!passed(&relative, "five-of-six"));
let bench = &field[0];
let five = per_run_passes(&field[2], Some(bench), &cfg);
assert_eq!(five, vec![true, true, true, true, true, false]);
let zero = per_run_passes(bench, Some(bench), &cfg);
assert_eq!(zero, vec![false; 6]);
let lax = ScoreConfig {
per_run_psr_bar: 0.5,
..cfg.clone()
};
assert_eq!(per_run_passes(bench, Some(bench), &lax), vec![false; 6]);
for d in &default {
let r = relative.iter().find(|s| s.agent_id == d.agent_id).unwrap();
assert_eq!(d.deflated_sharpe.to_bits(), r.deflated_sharpe.to_bits());
assert_eq!(d.psr.to_bits(), r.psr.to_bits());
assert_eq!(d.bootstrap_p.to_bits(), r.bootstrap_p.to_bits());
assert_eq!(
d.worst_run_drawdown.to_bits(),
r.worst_run_drawdown.to_bits()
);
assert_eq!(d.process_ok, r.process_ok);
}
}
#[test]
fn relative_verdict_without_a_benchmark_fails_every_run() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("absent");
assert!(rank(&field, &cfg).iter().all(|s| !s.passed_k));
let alone = score_agent(
&field[1],
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert!(!alone.passed_k);
let mut short = field[0].clone();
short.runs[2].returns.pop();
let v = per_run_passes(
&field[1],
Some(&short),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, false, true, true, true]);
let mut fewer = field[0].clone();
fewer.runs.truncate(4);
let v = per_run_passes(
&field[1],
Some(&fewer),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, true, true, false, false]);
}
#[test]
fn relative_preset_differs_only_in_mode_and_benchmark_id() {
let mut preset = serde_json::to_value(ScoreConfig::relative_to_benchmark("index")).unwrap();
let default = serde_json::to_value(ScoreConfig::default()).unwrap();
assert_eq!(preset["pass_mode"], "relative_to_benchmark");
assert_eq!(preset["benchmark_agent_id"], "index");
preset["pass_mode"] = default["pass_mode"].clone();
preset["benchmark_agent_id"] = default["benchmark_agent_id"].clone();
assert_eq!(preset, default);
}
#[test]
fn warn_events_lower_process_score_and_count_but_not_eligibility() {
let clean = agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let mut warned_runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
warned_runs[3]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
let cfg = ScoreConfig::default();
let c = score_agent(&clean, &cfg);
let w = score_agent(&warned, &cfg);
assert_eq!(c.process_warnings, 0);
assert_eq!(c.process_score, 1.0);
assert_eq!(w.process_warnings, 2);
assert!((w.process_score - 0.8).abs() < 1e-9);
assert!(c.process_ok && w.process_ok);
assert_eq!(c.rank_eligible, w.rank_eligible);
assert!(w.rank_eligible);
assert_eq!(c.deflated_sharpe.to_bits(), w.deflated_sharpe.to_bits());
assert_eq!(c.composite.to_bits(), w.composite.to_bits());
}
#[test]
fn block_violation_zeroes_graded_process_score() {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
runs[1].trace.events.push(ProcessEvent::DenylistBypass);
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
let s = score_agent(&agent("blocked", runs), &ScoreConfig::default());
assert_eq!(s.process_score, 0.0);
assert_eq!(s.process_warnings, 1, "warns are still counted");
assert!(!s.process_ok && !s.rank_eligible);
}
#[test]
fn tie_band_orders_cleaner_process_first() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let clean = agent("clean", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let mut warned_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
for field in [
vec![clean.clone(), warned.clone()],
vec![warned.clone(), clean.clone()],
] {
let board = rank(&field, &cfg);
assert_eq!(board[0].agent_id, "clean", "cleaner process leads the band");
assert_eq!(board[1].agent_id, "warned");
assert_eq!(board[0].rank_ordinal, 1);
assert_eq!(board[1].rank_ordinal, 2);
assert!(board[0].rank_eligible && board[1].rank_eligible);
assert_eq!(board[0].tie_group, board[1].tie_group);
assert!(board[0].dsr_tied && board[1].dsr_tied);
}
}
#[test]
fn process_tie_break_never_crosses_tie_bands() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let mut strong_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
strong_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let strong_warned = agent("strong_warned", strong_runs);
let weak_clean = agent("weak_clean", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[weak_clean, strong_warned], &cfg);
assert!(board.iter().all(|s| s.rank_eligible));
assert_eq!(board[0].agent_id, "strong_warned");
assert_ne!(board[0].tie_group, board[1].tie_group, "separable bands");
assert!(board[0].process_score < board[1].process_score);
}
#[test]
fn econ_rationality_reported_from_declared_candidates() {
let cfg = ScoreConfig::default();
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let none = score_agent(&agent("plain", runs.clone()), &cfg);
assert!(none.econ_rationality_score.is_none());
assert!(none.econ_dominance_violations.is_none());
let mut dominated = agent("dominated", runs.clone());
dominated.candidates = vec![(0..300)
.map(|i| 0.01 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let d = score_agent(&dominated, &cfg);
assert_eq!(d.econ_rationality_score, Some(0.0));
assert_eq!(d.econ_dominance_violations, Some(1));
let mut rational = agent("rational", runs);
rational.candidates = vec![(0..300)
.map(|i| 0.0001 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let r = score_agent(&rational, &cfg);
assert_eq!(r.econ_rationality_score, Some(1.0));
assert_eq!(r.econ_dominance_violations, Some(0));
assert_eq!(d.rank_eligible, r.rank_eligible);
}
#[test]
fn behavior_role_contributions_are_reported() {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
for r in &mut runs {
r.trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: true,
});
}
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
let s = score_agent(&agent("mixed", runs), &ScoreConfig::default());
let names: Vec<&str> = s
.role_contributions
.iter()
.map(|c| c.role.as_str())
.collect();
assert_eq!(names, vec!["clean_active", "warned"]);
let again = score_agent(
&agent("mixed", {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
for r in &mut runs {
r.trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: true,
});
}
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
runs
}),
&ScoreConfig::default(),
);
assert_eq!(s.role_contributions, again.role_contributions);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert!(empty.role_contributions.is_empty());
}
#[test]
fn archived_scores_without_new_fields_deserialize_with_defaults() {
let s = score_agent(
&agent("archived", (0..3).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
let mut v = serde_json::to_value(&s).expect("serialize");
let obj = v.as_object_mut().expect("object");
for field in [
"process_score",
"process_warnings",
"econ_rationality_score",
"econ_dominance_violations",
"role_contributions",
] {
assert!(obj.remove(field).is_some(), "{field} should be present");
}
let parsed: CompositeScore = serde_json::from_value(v).expect("archived JSON parses");
assert_eq!(parsed.process_score, 1.0);
assert_eq!(parsed.process_warnings, 0);
assert!(parsed.econ_rationality_score.is_none());
assert!(parsed.econ_dominance_violations.is_none());
assert!(parsed.role_contributions.is_empty());
}
#[test]
fn worst_run_drawdown_is_the_max_over_runs_not_the_pooled_track() {
let down = || Run {
returns: vec![-0.05; 4],
..Run::default()
};
let up = Run {
returns: vec![0.10; 10],
..Run::default()
};
let sub = agent("runs", vec![up.clone(), down(), down()]);
let s = score_agent(&sub, &ScoreConfig::default());
let expected = max_drawdown(&up.returns).max(max_drawdown(&down().returns));
assert_eq!(s.worst_run_drawdown.to_bits(), expected.to_bits());
assert!((s.worst_run_drawdown - (1.0 - 0.95f64.powi(4))).abs() < 1e-12);
assert_eq!(
s.max_drawdown.to_bits(),
max_drawdown(&pooled_of(&sub)).to_bits()
);
assert!((s.max_drawdown - (1.0 - 0.95f64.powi(8))).abs() < 1e-12);
assert!(s.max_drawdown > s.worst_run_drawdown);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert_eq!(empty.worst_run_drawdown, 0.0);
}
fn declare(pairs: &[(&str, DeclaredMandate)]) -> MandateDeclarations {
pairs
.iter()
.map(|(id, m)| (id.to_string(), m.clone()))
.collect()
}
fn without_declared(mut s: CompositeScore) -> CompositeScore {
s.declared_mandate = None;
s.verdict_applied = None;
s.declared_passed_k = None;
s.declared_mandate_eligible = None;
s.declared_mandate_ordinal = None;
s
}
#[test]
fn declaration_is_recorded_and_absent_by_default() {
let field = relative_field();
let cfg = ScoreConfig::default();
assert_eq!(
rank(&field, &cfg),
rank_declared(&field, &declare(&[]), &cfg)
);
let board = rank_declared(
&field,
&declare(&[
("beats", DeclaredMandate::OutperformBuyAndHold),
(
"trails",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.2,
},
),
]),
&cfg,
);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(
get("beats").declared_mandate,
Some(DeclaredMandate::OutperformBuyAndHold)
);
assert_eq!(
get("beats").verdict_applied,
Some(MandateVerdict::RelativeTo {
benchmark_id: "buy-and-hold".to_string()
})
);
assert_eq!(
get("trails").verdict_applied,
Some(MandateVerdict::DrawdownCapped {
max_per_run_drawdown: 0.2
})
);
let json = serde_json::to_string(get("beats")).unwrap();
assert!(json.contains("\"declared_mandate\":{\"kind\":\"outperform_buy_and_hold\"}"));
assert!(json.contains(
"\"verdict_applied\":{\"kind\":\"relative_to\",\"benchmark_id\":\"buy-and-hold\"}"
));
for id in ["buy-and-hold", "five-of-six", "clone"] {
let s = get(id);
assert!(s.declared_mandate.is_none() && s.verdict_applied.is_none());
assert!(s.declared_passed_k.is_none() && s.declared_mandate_eligible.is_none());
let json = serde_json::to_string(s).unwrap();
assert!(!json.contains("declared_") && !json.contains("verdict_applied"));
}
assert_eq!(
get("beats").mandate_verdict_label().as_deref(),
Some(if get("beats").declared_mandate_eligible == Some(true) {
"eligible under declared verdict (relative to buy-and-hold); host-board ineligible"
} else {
"ineligible under declared verdict (relative to buy-and-hold); host-board ineligible"
})
);
assert!(get("clone").mandate_verdict_label().is_none());
let wire = r#"[{"agent_id":"a","runs":[{"returns":[0.01,0.02]}],
"declared_mandate":{"kind":"drawdown_capped","max_per_run_drawdown":0.2}},
{"agent_id":"b","runs":[{"returns":[0.01,0.02]}]}]"#;
let parsed: Vec<DeclaredSubmission> = serde_json::from_str(wire).unwrap();
let (subs, decls) = split_declarations(parsed);
assert_eq!(subs.len(), 2);
assert_eq!(subs[0].runs[0].returns, vec![0.01, 0.02]);
assert_eq!(decls.len(), 1);
assert_eq!(
decls["a"],
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.2
}
);
}
#[test]
fn buy_and_hold_under_long_only_beta_gets_the_relative_verdict_and_fails_it() {
let field = relative_field();
let board = rank_declared(
&field,
&declare(&[
("buy-and-hold", DeclaredMandate::OutperformBuyAndHold),
("clone", DeclaredMandate::OutperformBuyAndHold),
("beats", DeclaredMandate::OutperformBuyAndHold),
]),
&ScoreConfig::default(),
);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let relative = Some(MandateVerdict::RelativeTo {
benchmark_id: "buy-and-hold".to_string(),
});
assert_eq!(get("buy-and-hold").verdict_applied, relative);
assert_eq!(get("buy-and-hold").declared_passed_k, Some(false));
assert_eq!(get("buy-and-hold").declared_mandate_eligible, Some(false));
assert_eq!(
get("buy-and-hold").mandate_verdict_label().as_deref(),
Some(
"ineligible under declared verdict (relative to buy-and-hold); host-board ineligible"
)
);
assert_eq!(get("clone").declared_passed_k, Some(false));
assert_eq!(get("beats").declared_passed_k, Some(true));
let plain = rank(&field, &ScoreConfig::default());
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(&without_declared(s.clone()), p, "{}", s.agent_id);
}
}
#[test]
fn a_declaration_cannot_flip_dsr_eligibility_or_move_the_host_columns() {
let field = field_with_one_single_run_failure();
let kinds = [
DeclaredMandate::AbsoluteReturn,
DeclaredMandate::OutperformBuyAndHold,
DeclaredMandate::RelativeTo {
benchmark_id: "a".to_string(),
},
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
];
let cfg = ScoreConfig::default();
let plain = rank(&field, &cfg);
for kind in &kinds {
let decls: MandateDeclarations = field
.iter()
.map(|s| (s.agent_id.clone(), kind.clone()))
.collect();
let board = rank_declared(&field, &decls, &cfg);
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(&without_declared(s.clone()), p, "{kind:?} {}", s.agent_id);
if s.declared_mandate_eligible == Some(true) {
assert!(s.deflated_sharpe >= cfg.dsr_bar);
assert!(s.bootstrap_p < cfg.alpha);
assert!(s.process_ok && s.mandate_ok);
}
}
if *kind == DeclaredMandate::AbsoluteReturn {
for s in &board {
assert_eq!(s.declared_mandate_eligible, Some(s.rank_eligible));
assert_eq!(s.declared_passed_k, Some(s.passed_k));
}
}
let strict = ScoreConfig {
dsr_bar: 10.0,
..ScoreConfig::default()
};
for s in rank_declared(&field, &decls, &strict) {
assert_eq!(s.declared_mandate_eligible, Some(false), "{kind:?}");
assert!(!s.rank_eligible);
}
}
let capped = rank_declared(
&field,
&declare(&[(
"one_bad_run",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
)]),
&cfg,
);
let bad = capped.iter().find(|s| s.agent_id == "one_bad_run").unwrap();
assert!(!bad.rank_eligible && !bad.passed_k);
assert_eq!(bad.declared_passed_k, Some(true));
assert_eq!(bad.declared_mandate_eligible, Some(true), "{bad:?}");
assert_eq!(
bad.mandate_verdict_label().as_deref(),
Some(
"eligible under declared verdict (drawdown capped at 0.50 per run); host-board ineligible"
)
);
}
#[test]
fn misdeclared_mandates_fail_closed() {
let field = relative_field();
let cfg = ScoreConfig::default();
let board = rank_declared(
&field,
&declare(&[(
"beats",
DeclaredMandate::RelativeTo {
benchmark_id: "absent".to_string(),
},
)]),
&cfg,
);
let beats = board.iter().find(|s| s.agent_id == "beats").unwrap();
assert_eq!(beats.declared_passed_k, Some(false));
assert_eq!(beats.declared_mandate_eligible, Some(false));
let ok = rank_declared(
&field,
&declare(&[("beats", DeclaredMandate::OutperformBuyAndHold)]),
&cfg,
);
assert_eq!(
ok.iter()
.find(|s| s.agent_id == "beats")
.unwrap()
.declared_passed_k,
Some(true)
);
let alone = score_agent_declared(
&field[1],
Some(&DeclaredMandate::OutperformBuyAndHold),
&cfg,
);
assert_eq!(alone.declared_passed_k, None);
assert_eq!(alone.declared_mandate_eligible, None);
assert_eq!(
score_agent_declared(&field[1], None, &cfg),
score_agent(&field[1], &cfg)
);
let clean = field_with_one_single_run_failure();
let capped = |x: f64| {
let board = rank_declared(
&clean,
&declare(&[(
"a",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: x,
},
)]),
&cfg,
);
let a = board.iter().find(|s| s.agent_id == "a").unwrap().clone();
(a.declared_passed_k, a.declared_mandate_eligible)
};
assert_eq!(capped(0.5), (Some(true), Some(true)));
for bad in [0.0, -0.2, 1.5, f64::NAN, f64::INFINITY] {
assert_eq!(capped(bad), (Some(true), Some(false)), "bound {bad}");
}
}
#[test]
fn declared_eligibility_is_ranked_within_its_mandate_class_only() {
let field = field_with_one_single_run_failure();
let cfg = ScoreConfig::default();
let plain = rank(&field, &cfg);
let board = rank_declared(
&field,
&declare(&[
(
"one_bad_run",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
),
("a", DeclaredMandate::AbsoluteReturn),
("b", DeclaredMandate::AbsoluteReturn),
]),
&cfg,
);
let order = |b: &[CompositeScore]| b.iter().map(|s| s.agent_id.clone()).collect::<Vec<_>>();
assert_eq!(order(&board), order(&plain));
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let bad = get("one_bad_run");
assert!(!bad.rank_eligible && bad.rank_ordinal == 0);
assert_eq!(bad.declared_mandate_eligible, Some(true));
assert_eq!(bad.declared_mandate_ordinal, Some(1));
let (a, b) = (get("a"), get("b"));
assert!(a.rank_eligible && b.rank_eligible);
let ords: Vec<Option<usize>> = [a, b].iter().map(|s| s.declared_mandate_ordinal).collect();
assert!(
ords.contains(&Some(1)) && ords.contains(&Some(2)),
"{ords:?}"
);
let (first, second) = if a.declared_mandate_ordinal == Some(1) {
(a, b)
} else {
(b, a)
};
assert!(
first.deflated_sharpe > second.deflated_sharpe
|| (first.deflated_sharpe == second.deflated_sharpe
&& first.agent_id < second.agent_id)
);
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(s.rank_ordinal, p.rank_ordinal);
}
}
}