use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
pub use sharpebench_protocol::DeclaredMandate;
use sharpebench_stats::StatisticalError;
use crate::calibration::brier_score;
use crate::certification::Certification;
use crate::comparison_sets::{comparison_set, restrict_to_shared, TaggedRun, TaggedSubmission};
use crate::decay::return_drift_half_life;
use crate::deflated_sharpe::{
checked_probabilistic_sharpe_ratio, deflated_sharpe_ratio_against_null, expected_max_sharpe,
observed_sharpe_ratio_of_windows, per_period_from_annualized,
};
use crate::econrationality::{elicit_revealed_selection, rationality_score};
use crate::pass_k::{pass_k, PassMode};
use crate::percentile::percentile_of;
use crate::process::{process_score, ProcessEvent, ProcessScore, Trace};
use crate::rediscovery::{clone_clusters, CLONE_COLLAPSE_COSINE};
use crate::roles::{attribute_behavior_roles, RoleContribution};
use crate::rolling::rolling_sharpe;
use crate::selection::{selection_robustness, SelectionRobustness};
use crate::significance::bootstrap_pvalue;
use crate::stats::{mean, std_dev};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Run {
pub returns: Vec<f64>,
#[serde(default)]
pub trace: Trace,
#[serde(default)]
pub confidences: Vec<f64>,
#[serde(default)]
pub outcomes: Vec<bool>,
#[serde(default)]
pub cost: f64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct AgentSubmission {
pub agent_id: String,
pub runs: Vec<Run>,
#[serde(default)]
pub in_sample_trials: u32,
#[serde(default)]
pub candidates: Vec<Vec<f64>>,
}
pub type MandateDeclarations = BTreeMap<String, DeclaredMandate>;
#[derive(Clone, Debug, Deserialize)]
pub struct DeclaredSubmission {
#[serde(flatten)]
pub submission: AgentSubmission,
#[serde(default)]
pub declared_mandate: Option<DeclaredMandate>,
}
pub fn split_declarations(
field: Vec<DeclaredSubmission>,
) -> (Vec<AgentSubmission>, MandateDeclarations) {
let mut declarations = MandateDeclarations::new();
let subs = field
.into_iter()
.map(|d| {
if let Some(m) = d.declared_mandate {
declarations.insert(d.submission.agent_id.clone(), m);
}
d.submission
})
.collect();
(subs, declarations)
}
pub fn parse_declared_field(
json: &str,
) -> Result<(Vec<AgentSubmission>, MandateDeclarations), String> {
let field: Vec<DeclaredSubmission> =
serde_json::from_str(json).map_err(|error| format!("invalid submissions JSON: {error}"))?;
let mut ids = std::collections::BTreeSet::new();
for item in &field {
let id = &item.submission.agent_id;
if id.trim().is_empty() || !ids.insert(id) {
return Err(format!(
"agent identity must be nonempty and unique: `{id}`"
));
}
}
Ok(split_declarations(field))
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum MandateVerdict {
AbsoluteReturn,
RelativeTo { benchmark_id: String },
DrawdownCapped { max_per_run_drawdown: f64 },
}
impl MandateVerdict {
pub fn of(declared: &DeclaredMandate) -> Self {
match declared {
DeclaredMandate::AbsoluteReturn => Self::AbsoluteReturn,
DeclaredMandate::RelativeTo { benchmark_id } => Self::RelativeTo {
benchmark_id: benchmark_id.clone(),
},
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown,
} => Self::DrawdownCapped {
max_per_run_drawdown: *max_per_run_drawdown,
},
DeclaredMandate::OutperformBuyAndHold => Self::RelativeTo {
benchmark_id: default_benchmark_agent_id(),
},
}
}
fn pass_mode(&self) -> PassMode {
match self {
Self::AbsoluteReturn => PassMode::All,
Self::RelativeTo { .. } => PassMode::RelativeToBenchmark,
Self::DrawdownCapped { .. } => PassMode::Any,
}
}
pub fn benchmark_id(&self) -> Option<&str> {
match self {
Self::RelativeTo { benchmark_id } => Some(benchmark_id),
Self::AbsoluteReturn | Self::DrawdownCapped { .. } => None,
}
}
fn drawdown_bound_holds(&self, worst_run_drawdown: f64) -> bool {
match self {
Self::DrawdownCapped {
max_per_run_drawdown: x,
} => *x > 0.0 && *x <= 1.0 && worst_run_drawdown <= *x,
Self::AbsoluteReturn | Self::RelativeTo { .. } => true,
}
}
pub fn describe(&self) -> String {
match self {
Self::AbsoluteReturn => "absolute return".to_string(),
Self::RelativeTo { benchmark_id } => format!("relative to {benchmark_id}"),
Self::DrawdownCapped {
max_per_run_drawdown,
} => format!("drawdown capped at {max_per_run_drawdown:.2} per run"),
}
}
}
fn class_key(v: &MandateVerdict) -> (u8, &str, u64) {
match v {
MandateVerdict::AbsoluteReturn => (0, "", 0),
MandateVerdict::RelativeTo { benchmark_id } => (1, benchmark_id, 0),
MandateVerdict::DrawdownCapped {
max_per_run_drawdown,
} => (2, "", max_per_run_drawdown.to_bits()),
}
}
struct ResolvedDeclaration<'a> {
mandate: &'a DeclaredMandate,
benchmark: Option<&'a AgentSubmission>,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub enum RankKey {
#[default]
DeflatedSharpe,
Alpha,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Mandate {
pub max_drawdown: f64,
#[serde(default = "default_max_run_drawdown")]
pub max_run_drawdown: f64,
}
impl Default for Mandate {
fn default() -> Self {
Self {
max_drawdown: 1.0,
max_run_drawdown: default_max_run_drawdown(),
}
}
}
fn default_max_run_drawdown() -> f64 {
1.0
}
fn max_drawdown(returns: &[f64]) -> f64 {
let mut nav = 1.0;
let mut peak = 1.0;
let mut mdd = 0.0;
for &r in returns {
nav *= 1.0 + r;
if nav > peak {
peak = nav;
}
if peak > 0.0 {
let dd = 1.0 - nav / peak;
if dd > mdd {
mdd = dd;
}
}
}
mdd
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ScoreConfig {
pub n_trials: u32,
pub trials_sr_std: f64,
#[serde(default)]
pub deflation_null_mean_per_period: f64,
pub dsr_bar: f64,
pub per_run_psr_bar: f64,
#[serde(default)]
pub per_run_min_annual_sharpe: f64,
#[serde(default)]
pub pass_mode: PassMode,
#[serde(default = "default_benchmark_agent_id")]
pub benchmark_agent_id: String,
#[serde(default = "default_periods_per_year")]
pub periods_per_year: f64,
pub alpha: f64,
pub bootstrap_seed: u64,
pub n_boot: usize,
pub block_prob: f64,
#[serde(default)]
pub mandate: Mandate,
#[serde(default)]
pub rank_key: RankKey,
#[serde(default)]
pub reference_dsr_population: Vec<f64>,
#[serde(default = "default_rolling_window")]
pub rolling_window: usize,
#[serde(default = "default_dsr_ci_level")]
pub dsr_ci_level: f64,
#[serde(default = "default_shared_run_set")]
pub shared_run_set: bool,
#[serde(default = "default_min_field_for_measured_sr_std")]
pub min_field_for_measured_sr_std: usize,
#[serde(default = "default_dedup_clones_for_measured_sr_std")]
pub dedup_clones_for_measured_sr_std: bool,
#[serde(default = "default_min_measured_trials_sr_std")]
pub min_measured_trials_sr_std: f64,
#[serde(default = "default_execution_seeds_per_window")]
pub execution_seeds_per_window: usize,
}
fn default_rolling_window() -> usize {
21
}
fn default_dsr_ci_level() -> f64 {
0.90
}
fn default_shared_run_set() -> bool {
true
}
fn default_min_field_for_measured_sr_std() -> usize {
5
}
fn default_dedup_clones_for_measured_sr_std() -> bool {
true
}
fn default_min_measured_trials_sr_std() -> f64 {
0.5
}
fn default_execution_seeds_per_window() -> usize {
1
}
fn default_periods_per_year() -> f64 {
252.0
}
fn default_benchmark_agent_id() -> String {
"buy-and-hold".to_string()
}
pub fn per_run_passes(
sub: &AgentSubmission,
benchmark: Option<&AgentSubmission>,
cfg: &ScoreConfig,
) -> Vec<bool> {
let bar = per_run_psr_benchmark(cfg);
match cfg.pass_mode {
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => sub
.runs
.iter()
.map(|r| {
checked_probabilistic_sharpe_ratio(&r.returns, bar)
.is_ok_and(|psr| psr >= cfg.per_run_psr_bar)
})
.collect(),
PassMode::RelativeToBenchmark => sub
.runs
.iter()
.enumerate()
.map(|(i, r)| {
benchmark
.and_then(|b| b.runs.get(i))
.and_then(|b| excess_returns(&r.returns, &b.returns))
.is_some_and(|e| {
std_dev(&e) > 0.0
&& checked_probabilistic_sharpe_ratio(&e, bar)
.is_ok_and(|psr| psr >= cfg.per_run_psr_bar)
})
})
.collect(),
}
}
fn excess_returns(returns: &[f64], benchmark: &[f64]) -> Option<Vec<f64>> {
(returns.len() == benchmark.len())
.then(|| returns.iter().zip(benchmark).map(|(a, b)| a - b).collect())
}
pub fn per_period_sr_std(cfg: &ScoreConfig) -> f64 {
per_period_from_annualized(cfg.trials_sr_std, cfg.periods_per_year)
}
fn checked_periods_per_year(cfg: &ScoreConfig) -> Result<(), StatisticalError> {
if !cfg.periods_per_year.is_finite() || cfg.periods_per_year <= 0.0 {
return Err(StatisticalError::InvalidParameter {
name: "periods_per_year",
requirement: "must be finite and positive",
});
}
Ok(())
}
pub fn per_run_psr_benchmark(cfg: &ScoreConfig) -> f64 {
per_period_from_annualized(cfg.per_run_min_annual_sharpe, cfg.periods_per_year)
}
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TrialsSrStdSource {
#[default]
Configured,
Measured,
MeasuredFloored,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct DispersionLeverage {
pub agent_id: String,
pub agents_in_vote: usize,
pub vote_sharpe: f64,
pub votes: usize,
pub measured_dispersion_without_it: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub leverage: Option<f64>,
}
impl Default for ScoreConfig {
fn default() -> Self {
Self {
n_trials: 50,
trials_sr_std: 0.5,
deflation_null_mean_per_period: 0.0,
dsr_bar: 0.95,
per_run_psr_bar: 0.90,
per_run_min_annual_sharpe: 0.0,
pass_mode: PassMode::default(),
benchmark_agent_id: default_benchmark_agent_id(),
periods_per_year: default_periods_per_year(),
alpha: 0.05,
bootstrap_seed: 0x5BA7_2026,
n_boot: 2000,
block_prob: 0.1,
mandate: Mandate::default(),
rank_key: RankKey::default(),
reference_dsr_population: Vec::new(),
rolling_window: default_rolling_window(),
dsr_ci_level: default_dsr_ci_level(),
shared_run_set: default_shared_run_set(),
min_field_for_measured_sr_std: default_min_field_for_measured_sr_std(),
dedup_clones_for_measured_sr_std: default_dedup_clones_for_measured_sr_std(),
min_measured_trials_sr_std: default_min_measured_trials_sr_std(),
execution_seeds_per_window: default_execution_seeds_per_window(),
}
}
}
impl ScoreConfig {
pub fn for_periods_per_year(periods_per_year: f64) -> Self {
Self {
periods_per_year,
..Self::default()
}
}
pub fn reliability_never_catastrophic(max_run_dd: f64) -> Self {
Self {
pass_mode: PassMode::Any,
mandate: Mandate {
max_run_drawdown: max_run_dd,
..Mandate::default()
},
..Self::default()
}
}
pub fn relative_to_benchmark(benchmark_agent_id: &str) -> Self {
Self {
pass_mode: PassMode::RelativeToBenchmark,
benchmark_agent_id: benchmark_agent_id.to_string(),
..Self::default()
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct CompositeScore {
pub agent_id: String,
pub deflated_sharpe: f64,
pub psr: f64,
pub passed_k: bool,
pub process_ok: bool,
pub bootstrap_p: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bootstrap_error: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub deflation_error: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selection_error: Option<String>,
pub raw_mean_return: f64,
pub rank_eligible: bool,
pub composite: f64,
pub alpha: f64,
pub beta: f64,
pub calibration_brier: Option<f64>,
#[serde(default)]
pub calibration_observations: usize,
pub edge_half_life: Option<f64>,
pub field_reality_check_p: f64,
pub max_drawdown: f64,
pub mandate_ok: bool,
#[serde(default)]
pub worst_run_drawdown: f64,
pub turnover: f64,
pub pareto_optimal: bool,
pub step_down_significant: bool,
pub confidence_weighted_return: f64,
pub cost: f64,
pub return_per_cost: Option<f64>,
pub field_spa_p: f64,
pub field_spa_consistent_p: f64,
#[serde(default)]
pub field_significance_benchmark: String,
pub field_crowdedness: Option<f64>,
#[serde(default)]
pub field_crowdedness_peers: usize,
pub in_sample_trials: u32,
pub effective_n_trials: u32,
pub dsr_percentile: Option<f64>,
pub selection_median_dsr: Option<f64>,
pub selection_gap: Option<f64>,
pub rank_ordinal: usize,
pub rolling_min_sharpe: Option<f64>,
pub rolling_frac_positive: Option<f64>,
#[serde(default)]
pub rolling_windows: usize,
pub sortino: Option<f64>,
pub downside_deviation: f64,
pub dsr_per_cost: Option<f64>,
pub process_floored: bool,
pub realized_floored_return: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dsr_ci_low: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dsr_ci_high: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dsr_se: Option<f64>,
pub tie_group: usize,
pub dsr_tied: bool,
#[serde(default)]
pub trials_sr_std: f64,
#[serde(default)]
pub trials_sr_std_annualized: Option<f64>,
#[serde(default)]
pub trials_sr_std_annualized_equivalent: f64,
#[serde(default)]
pub deflation_bar_per_period: f64,
#[serde(default)]
pub deflation_bar_annualized_equivalent: f64,
#[serde(default)]
pub deflation_null_mean_per_period: f64,
#[serde(default)]
pub pooled_observations: usize,
#[serde(default)]
pub trials_sr_std_source: TrialsSrStdSource,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trials_sr_std_most_influential_vote: Option<DispersionLeverage>,
#[serde(default)]
pub runs_submitted: usize,
#[serde(default)]
pub runs_scored: usize,
#[serde(default = "default_process_score")]
pub process_score: f64,
#[serde(default)]
pub process_warnings: usize,
#[serde(default)]
pub econ_rationality_score: Option<f64>,
#[serde(default)]
pub econ_dominance_violations: Option<usize>,
#[serde(default)]
pub role_contributions: Vec<RoleContribution>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate: Option<DeclaredMandate>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub verdict_applied: Option<MandateVerdict>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_passed_k: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate_eligible: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_mandate_ordinal: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub certification: Option<Certification>,
}
impl CompositeScore {
pub fn mandate_verdict_label(&self) -> Option<String> {
let verdict = self.verdict_applied.as_ref()?;
let eligibility = if self.declared_mandate_eligible == Some(true) {
"eligible"
} else {
"ineligible"
};
let host = if self.rank_eligible {
"host-board eligible"
} else {
"host-board ineligible"
};
Some(format!(
"{eligibility} under declared verdict ({}); {host}",
verdict.describe()
))
}
}
fn default_process_score() -> f64 {
1.0
}
fn dominates(a: &CompositeScore, b: &CompositeScore) -> bool {
a.raw_mean_return >= b.raw_mean_return
&& a.max_drawdown <= b.max_drawdown
&& a.turnover <= b.turnover
&& (a.raw_mean_return > b.raw_mean_return
|| a.max_drawdown < b.max_drawdown
|| a.turnover < b.turnover)
}
fn has_no_sharpe_ratio(tracks: &[Vec<f64>]) -> bool {
observed_sharpe_ratio_of_windows(&window_slices(tracks)).is_err()
}
#[derive(Clone, Copy)]
struct Deflation {
sr_std: f64,
annualized: Option<f64>,
source: TrialsSrStdSource,
null_mean_per_period: f64,
}
impl Deflation {
fn configured(cfg: &ScoreConfig) -> Self {
Self {
sr_std: per_period_sr_std(cfg),
annualized: Some(cfg.trials_sr_std),
source: TrialsSrStdSource::Configured,
null_mean_per_period: cfg.deflation_null_mean_per_period,
}
}
fn measured(sr_std: f64, cfg: &ScoreConfig) -> Self {
let floor =
per_period_from_annualized(cfg.min_measured_trials_sr_std, cfg.periods_per_year);
let floored = sr_std < floor;
Self {
sr_std: sr_std.max(floor),
annualized: None,
source: if floored {
TrialsSrStdSource::MeasuredFloored
} else {
TrialsSrStdSource::Measured
},
null_mean_per_period: cfg.deflation_null_mean_per_period,
}
}
}
fn stated_confidence(run: &Run) -> Option<f64> {
if run.confidences.is_empty() {
return None;
}
let total: f64 = run.confidences.iter().map(|c| c.clamp(0.0, 1.0)).sum();
Some(total / run.confidences.len() as f64)
}
pub fn score_agent(sub: &AgentSubmission, cfg: &ScoreConfig) -> CompositeScore {
score_agent_with(sub, &sub.runs, cfg, Deflation::configured(cfg), None, None)
}
pub fn score_agent_declared(
sub: &AgentSubmission,
declared: Option<&DeclaredMandate>,
cfg: &ScoreConfig,
) -> CompositeScore {
if let Some(mandate) = declared {
let verdict = MandateVerdict::of(mandate);
if verdict.benchmark_id().is_some() {
let mut score = score_agent(sub, cfg);
score.declared_mandate = Some(mandate.clone());
score.verdict_applied = Some(verdict);
return score;
}
}
let resolved = declared.map(|mandate| ResolvedDeclaration {
mandate,
benchmark: None,
});
score_agent_with(
sub,
&sub.runs,
cfg,
Deflation::configured(cfg),
None,
resolved,
)
}
fn score_agent_with(
sub: &AgentSubmission,
process_runs: &[Run],
cfg: &ScoreConfig,
defl: Deflation,
benchmark: Option<&AgentSubmission>,
declared: Option<ResolvedDeclaration<'_>>,
) -> CompositeScore {
let tracks = window_tracks(sub, cfg.execution_seeds_per_window);
let pooled = tracks.concat();
let pooled_psr = checked_probabilistic_sharpe_ratio(&pooled, 0.0);
let sharpe_undefined = matches!(
pooled_psr,
Err(StatisticalError::InvalidParameter {
name: "returns",
..
} | StatisticalError::NonFiniteComputation {
quantity: "Sharpe ratio"
})
);
let psr = pooled_psr.unwrap_or(0.0);
let effective_n_trials = cfg.n_trials.saturating_add(sub.in_sample_trials);
let frequency = checked_periods_per_year(cfg);
let deflation = frequency
.and_then(|()| observed_sharpe_ratio_of_windows(&window_slices(&tracks)).map(|_| ()))
.and_then(|()| expected_max_sharpe(defl.sr_std, effective_n_trials))
.and_then(|bar| {
let dsr = deflated_sharpe_ratio_against_null(
&pooled,
effective_n_trials,
defl.null_mean_per_period,
defl.sr_std,
)?;
Ok((dsr, defl.null_mean_per_period + bar))
});
let deflation_error = deflation.as_ref().err().map(ToString::to_string);
let (dsr, deflation_bar_per_period) = deflation.unwrap_or((0.0, 0.0));
let per_run = per_run_passes(sub, benchmark, cfg);
let passed_k = pass_k(&per_run, cfg.pass_mode);
let per_run_process: Vec<ProcessScore> = process_runs
.iter()
.map(|r| process_score(&r.trace))
.collect();
let process_ok = per_run_process.iter().all(ProcessScore::is_clean);
let process_warnings: usize = per_run_process.iter().map(|p| p.warn_violations).sum();
let graded_process_score = if process_ok {
(1.0 - process_warnings as f64 * 0.1).max(0.0)
} else {
0.0
};
let bootstrap = bootstrap_pvalue(&pooled, cfg.bootstrap_seed, cfg.n_boot, cfg.block_prob);
let bootstrap_error = bootstrap.as_ref().err().map(ToString::to_string);
let bootstrap_p = bootstrap.unwrap_or(1.0);
let raw_mean_return = mean(&pooled);
let calibration_pairs: Vec<(f64, bool)> = sub
.runs
.iter()
.flat_map(|run| {
run.confidences
.iter()
.copied()
.zip(run.outcomes.iter().copied())
})
.collect();
let calibration_observations = calibration_pairs.len();
let (conf, outc): (Vec<f64>, Vec<bool>) = calibration_pairs.into_iter().unzip();
let calibration_brier = if calibration_observations > 0 {
Some(brier_score(&conf, &outc))
} else {
None
};
let seeds_per_window = cfg.execution_seeds_per_window.max(1);
let per_window_edge: Vec<f64> = sub
.runs
.chunks(seeds_per_window)
.map(|window| window.iter().map(|r| mean(&r.returns)).sum::<f64>() / window.len() as f64)
.collect();
let edge_half_life_periods = return_drift_half_life(&per_window_edge);
let mdd = max_drawdown(&pooled);
let worst_run_drawdown = sub
.runs
.iter()
.map(|r| max_drawdown(&r.returns))
.fold(0.0, f64::max);
let mandate_ok =
mdd <= cfg.mandate.max_drawdown && worst_run_drawdown <= cfg.mandate.max_run_drawdown;
let total_orders: usize = sub
.runs
.iter()
.map(|r| {
r.trace
.events
.iter()
.filter(|e| matches!(e, ProcessEvent::OrderPlaced { .. }))
.count()
})
.sum();
let turnover = total_orders as f64 / sub.runs.len().max(1) as f64;
let stated_weights: Vec<f64> = sub.runs.iter().filter_map(stated_confidence).collect();
let unstated_weight = if stated_weights.is_empty() {
1.0
} else {
mean(&stated_weights)
};
let mut cw_num = 0.0;
let mut cw_den = 0.0;
for r in &sub.runs {
let w = stated_confidence(r).unwrap_or(unstated_weight);
cw_num += w * mean(&r.returns);
cw_den += w;
}
let confidence_weighted_return = if cw_den > 0.0 {
cw_num / cw_den
} else {
raw_mean_return
};
let cost: f64 = sub.runs.iter().map(|r| r.cost).sum();
let return_per_cost = if cost > 0.0 {
Some(raw_mean_return / cost)
} else {
None
};
let dsr_percentile = if cfg.reference_dsr_population.is_empty() || sharpe_undefined {
None
} else {
Some(percentile_of(dsr, &cfg.reference_dsr_population))
};
let selection =
if sub.candidates.is_empty() {
None
} else {
Some(frequency.and_then(|()| {
selection_robustness(&sub.candidates, effective_n_trials, defl.sr_std)
}))
};
let selection_error = selection
.as_ref()
.and_then(|s: &Result<SelectionRobustness, _>| s.as_ref().err())
.map(ToString::to_string);
let (selection_median_dsr, selection_gap) = match selection.and_then(Result::ok) {
Some(sr) => (Some(sr.median_dsr), Some(sr.selection_gap)),
None => (None, None),
};
let rolling = rolling_sharpe(&pooled, cfg.rolling_window);
let rolling_summary = rolling.filter(|_| !sharpe_undefined);
let rolling_min_sharpe = rolling_summary.map(|r| r.min_sharpe);
let rolling_frac_positive = rolling_summary.map(|r| r.frac_positive);
let rolling_windows = rolling.map_or(0, |r| r.n_windows);
let sortino = crate::stats::sortino_ratio(&pooled, 0.0);
let downside_deviation = crate::stats::downside_deviation(&pooled, 0.0);
let dsr_per_cost = if cost > 0.0 && !sharpe_undefined {
Some(dsr / cost)
} else {
None
};
let process_floored = !process_ok;
let realized_floored_return = if process_floored {
0.0
} else {
raw_mean_return
};
let dsr_ci = frequency.and_then(|()| {
crate::significance::bootstrap_dsr_ci_against_null(
&pooled,
effective_n_trials,
defl.null_mean_per_period,
defl.sr_std,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.dsr_ci_level,
)
});
let dsr_ci_error = dsr_ci.as_ref().err().map(ToString::to_string);
let dsr_ci = dsr_ci.ok();
let deflation_error = deflation_error.or(dsr_ci_error);
let econ_choice = if sharpe_undefined {
None
} else {
elicit_revealed_selection(&sub.candidates, &pooled)
};
let (econ_rationality_score, econ_dominance_violations) = match econ_choice {
Some(choice) => {
let violations = usize::from(choice.is_dominated());
(Some(rationality_score(&[choice])), Some(violations))
}
None => (None, None),
};
let role_contributions = attribute_behavior_roles(&sub.runs, seeds_per_window);
let rank_eligible = dsr >= cfg.dsr_bar
&& passed_k
&& process_ok
&& bootstrap_error.is_none()
&& deflation_error.is_none()
&& bootstrap_p < cfg.alpha
&& mandate_ok;
let composite = if rank_eligible { dsr } else { 0.0 };
let declared = declared.map(|d| {
let verdict = MandateVerdict::of(d.mandate);
let verdict_cfg = ScoreConfig {
pass_mode: verdict.pass_mode(),
benchmark_agent_id: verdict
.benchmark_id()
.map_or_else(|| cfg.benchmark_agent_id.clone(), str::to_string),
..cfg.clone()
};
let per_run = per_run_passes(sub, d.benchmark, &verdict_cfg);
let declared_passed_k = pass_k(&per_run, verdict_cfg.pass_mode);
let eligible = dsr >= cfg.dsr_bar
&& declared_passed_k
&& process_ok
&& bootstrap_error.is_none()
&& deflation_error.is_none()
&& bootstrap_p < cfg.alpha
&& mandate_ok
&& verdict.drawdown_bound_holds(worst_run_drawdown);
(d.mandate.clone(), verdict, declared_passed_k, eligible)
});
let (declared_mandate, verdict_applied, declared_passed_k, declared_mandate_eligible) =
match declared {
Some((m, v, p, e)) => (Some(m), Some(v), Some(p), Some(e)),
None => (None, None, None, None),
};
CompositeScore {
agent_id: sub.agent_id.clone(),
deflated_sharpe: dsr,
psr,
passed_k,
process_ok,
bootstrap_p,
bootstrap_error,
deflation_error,
selection_error,
raw_mean_return,
rank_eligible,
composite,
alpha: 0.0,
beta: 0.0,
calibration_brier,
calibration_observations,
edge_half_life: edge_half_life_periods,
field_reality_check_p: 1.0,
max_drawdown: mdd,
mandate_ok,
worst_run_drawdown,
turnover,
pareto_optimal: false,
step_down_significant: false,
confidence_weighted_return,
cost,
return_per_cost,
field_spa_p: 1.0,
field_spa_consistent_p: 1.0,
field_significance_benchmark: "unscored".to_string(),
field_crowdedness: None,
field_crowdedness_peers: 0,
in_sample_trials: sub.in_sample_trials,
effective_n_trials,
dsr_percentile,
selection_median_dsr,
selection_gap,
rank_ordinal: 0,
rolling_min_sharpe,
rolling_frac_positive,
rolling_windows,
sortino,
downside_deviation,
dsr_per_cost,
process_floored,
realized_floored_return,
dsr_ci_low: dsr_ci.map(|c| c.lower),
dsr_ci_high: dsr_ci.map(|c| c.upper),
dsr_se: dsr_ci.map(|c| c.se),
tie_group: 0,
dsr_tied: false,
trials_sr_std: defl.sr_std,
trials_sr_std_annualized: defl.annualized,
trials_sr_std_annualized_equivalent: defl.sr_std * cfg.periods_per_year.sqrt(),
deflation_bar_per_period,
deflation_bar_annualized_equivalent: deflation_bar_per_period * cfg.periods_per_year.sqrt(),
deflation_null_mean_per_period: defl.null_mean_per_period,
pooled_observations: pooled.len(),
trials_sr_std_source: defl.source,
trials_sr_std_most_influential_vote: None,
runs_submitted: sub.runs.len(),
runs_scored: sub.runs.len(),
process_score: graded_process_score,
process_warnings,
econ_rationality_score,
econ_dominance_violations,
role_contributions,
declared_mandate,
verdict_applied,
declared_passed_k,
declared_mandate_eligible,
declared_mandate_ordinal: None,
certification: None,
}
}
pub fn pooled_returns(sub: &AgentSubmission, seeds_per_window: usize) -> Vec<f64> {
window_tracks(sub, seeds_per_window).concat()
}
pub fn window_tracks(sub: &AgentSubmission, seeds_per_window: usize) -> Vec<Vec<f64>> {
let width = seeds_per_window.max(1);
if width == 1 {
return sub.runs.iter().map(|r| r.returns.clone()).collect();
}
assert!(
sub.runs.len().is_multiple_of(width),
"{} runs cannot form complete {}-execution window blocks",
sub.runs.len(),
width
);
sub.runs
.chunks_exact(width)
.map(|replicates| {
let len = replicates.first().map_or(0, |r| r.returns.len());
assert!(
replicates.iter().all(|r| r.returns.len() == len),
"execution replicates in one window must have equal return lengths"
);
(0..len)
.map(|t| {
replicates.iter().map(|r| r.returns[t]).sum::<f64>() / replicates.len() as f64
})
.collect()
})
.collect()
}
fn window_slices(tracks: &[Vec<f64>]) -> Vec<&[f64]> {
tracks.iter().map(Vec::as_slice).collect()
}
pub(crate) fn restrict_to_shared_positions(subs: &[AgentSubmission]) -> Vec<AgentSubmission> {
let tag = |i: usize| format!("{i:08}");
let tagged: Vec<TaggedSubmission> = subs
.iter()
.filter(|s| !s.runs.is_empty())
.map(|s| TaggedSubmission {
agent_id: s.agent_id.clone(),
runs: s
.runs
.iter()
.enumerate()
.map(|(i, run)| TaggedRun {
window_id: tag(i),
run: run.clone(),
})
.collect(),
in_sample_trials: s.in_sample_trials,
candidates: s.candidates.clone(),
})
.collect();
let roster: Vec<String> = tagged.iter().map(|s| s.agent_id.clone()).collect();
let set = comparison_set(&roster, &tagged);
let mut tagged_iter = tagged.iter();
subs.iter()
.map(|s| {
if s.runs.is_empty() {
return s.clone();
}
let t = tagged_iter
.next()
.expect("one tagged submission per non-empty submission");
restrict_to_shared(&set, t)
})
.collect()
}
fn measured_trials_sr_std(
tracks: &[Vec<Vec<f64>>],
ids: &[&str],
min_field: usize,
dedup_clones: bool,
) -> Option<MeasuredDispersion> {
let qualifying: Vec<(usize, Vec<f64>, f64)> = tracks
.iter()
.enumerate()
.filter_map(|(index, t)| {
let sharpe = observed_sharpe_ratio_of_windows(&window_slices(t)).ok()?;
Some((index, t.concat(), sharpe))
})
.collect();
let mut votes: Vec<(usize, usize, f64)> = if dedup_clones {
let streams: Vec<Vec<f64>> = qualifying.iter().map(|(_, p, _)| p.clone()).collect();
clone_clusters(&streams, CLONE_COLLAPSE_COSINE, false)
.iter()
.map(|members| {
let mut cluster: Vec<(usize, f64)> = members
.iter()
.map(|&i| (qualifying[i].0, qualifying[i].2))
.collect();
cluster.sort_by(|a, b| {
a.1.partial_cmp(&b.1)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| ids[a.0].cmp(ids[b.0]))
});
let (index, sharpe) = cluster[(cluster.len() - 1) / 2];
(index, members.len(), sharpe)
})
.collect()
} else {
qualifying
.iter()
.map(|&(index, _, sharpe)| (index, 1, sharpe))
.collect()
};
if votes.len() < min_field.max(2) {
return None;
}
votes.sort_by(|a, b| {
a.2.partial_cmp(&b.2)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| ids[a.0].cmp(ids[b.0]))
});
let sharpes: Vec<f64> = votes.iter().map(|v| v.2).collect();
let sr_std = std_dev(&sharpes);
Some(MeasuredDispersion {
sr_std,
most_influential_vote: most_influential_vote(&votes, &sharpes, sr_std, ids),
})
}
struct MeasuredDispersion {
sr_std: f64,
most_influential_vote: Option<DispersionLeverage>,
}
fn most_influential_vote(
votes: &[(usize, usize, f64)],
sharpes: &[f64],
sr_std: f64,
ids: &[&str],
) -> Option<DispersionLeverage> {
let equal = sharpes.first() == sharpes.last();
if equal || !sr_std.is_finite() || sr_std <= 0.0 {
return None;
}
let without = |skip: usize| -> f64 {
let rest: Vec<f64> = sharpes
.iter()
.enumerate()
.filter(|&(i, _)| i != skip)
.map(|(_, &x)| x)
.collect();
std_dev(&rest)
};
let outranks = |a: (Option<f64>, &str), b: (Option<f64>, &str)| -> bool {
match (a.0, b.0) {
(None, Some(_)) => true,
(Some(_), None) => false,
(Some(x), Some(y)) if x != y => x > y,
_ => a.1 < b.1,
}
};
let mut best: Option<(usize, f64, Option<f64>)> = None;
for position in 0..votes.len() {
let rest = without(position);
let leverage = (rest > 0.0).then(|| sr_std / rest);
let candidate = (leverage, ids[votes[position].0]);
let beats = best.is_none_or(|(held, _, held_leverage)| {
outranks(candidate, (held_leverage, ids[votes[held].0]))
});
if beats {
best = Some((position, rest, leverage));
}
}
let (position, rest, leverage) = best.expect("a measured field has at least two votes");
let (index, agents_in_vote, vote_sharpe) = votes[position];
Some(DispersionLeverage {
agent_id: ids[index].to_string(),
agents_in_vote,
vote_sharpe,
votes: votes.len(),
measured_dispersion_without_it: rest,
leverage,
})
}
pub fn rank(subs: &[AgentSubmission], cfg: &ScoreConfig) -> Vec<CompositeScore> {
rank_declared(subs, &MandateDeclarations::new(), cfg)
}
pub fn rank_declared(
subs: &[AgentSubmission],
declarations: &MandateDeclarations,
cfg: &ScoreConfig,
) -> Vec<CompositeScore> {
let restricted;
let field: &[AgentSubmission] = if cfg.shared_run_set {
restricted = restrict_to_shared_positions(subs);
&restricted
} else {
subs
};
let mut rank_cfg = cfg.clone();
rank_cfg.n_trials = cfg
.n_trials
.max(u32::try_from(field.len()).unwrap_or(u32::MAX));
let tracks: Vec<Vec<Vec<f64>>> = field
.iter()
.map(|s| window_tracks(s, rank_cfg.execution_seeds_per_window))
.collect();
let pooled: Vec<Vec<f64>> = tracks.iter().map(|t| t.concat()).collect();
let min_len = pooled.iter().map(Vec::len).min().unwrap_or(0);
let n_agents = pooled.len().max(1) as f64;
let market: Vec<f64> = (0..min_len)
.map(|i| pooled.iter().map(|p| p[i]).sum::<f64>() / n_agents)
.collect();
let (significance_benchmark, significance_benchmark_label): (Vec<f64>, String) = field
.iter()
.position(|s| s.agent_id == rank_cfg.benchmark_agent_id)
.filter(|&idx| pooled[idx].len() >= min_len)
.map(|idx| {
(
pooled[idx][..min_len].to_vec(),
rank_cfg.benchmark_agent_id.clone(),
)
})
.unwrap_or_else(|| (vec![0.0; min_len], "zero-return-cash".to_string()));
let ids: Vec<&str> = field.iter().map(|s| s.agent_id.as_str()).collect();
let measured = measured_trials_sr_std(
&tracks,
&ids,
cfg.min_field_for_measured_sr_std,
cfg.dedup_clones_for_measured_sr_std,
);
let defl = measured.as_ref().map_or_else(
|| Deflation::configured(cfg),
|m| Deflation::measured(m.sr_std, cfg),
);
let most_influential_vote = measured.and_then(|m| m.most_influential_vote);
let benchmark = match cfg.pass_mode {
PassMode::RelativeToBenchmark => {
field.iter().find(|s| s.agent_id == cfg.benchmark_agent_id)
}
PassMode::All | PassMode::Any | PassMode::AtLeast(_) => None,
};
let mut scores: Vec<CompositeScore> = field
.iter()
.enumerate()
.map(|(idx, s)| {
let declared = declarations
.get(&s.agent_id)
.map(|mandate| ResolvedDeclaration {
mandate,
benchmark: MandateVerdict::of(mandate)
.benchmark_id()
.and_then(|id| field.iter().find(|b| b.agent_id == id)),
});
let mut cs = score_agent_with(s, &subs[idx].runs, &rank_cfg, defl, benchmark, declared);
cs.runs_submitted = subs[idx].runs.len();
cs.field_significance_benchmark = significance_benchmark_label.clone();
if min_len >= 2 {
let (alpha, beta) = crate::attribution::alpha_beta(&pooled[idx], &market);
cs.alpha = alpha;
cs.beta = beta;
}
cs
})
.collect();
if min_len >= 2 {
let field_excess: Vec<Vec<f64>> = pooled
.iter()
.map(|p| {
p.iter()
.take(min_len)
.zip(significance_benchmark.iter())
.map(|(a, b)| a - b)
.collect()
})
.collect();
let rc_p = crate::significance::reality_check_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_p = crate::significance::spa_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let spa_c_p = crate::significance::spa_consistent_pvalue(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
);
let sd = crate::significance::step_down_significant(
&field_excess,
cfg.bootstrap_seed,
cfg.n_boot,
cfg.block_prob,
cfg.alpha,
);
let field_error = rc_p
.as_ref()
.err()
.or(spa_p.as_ref().err())
.or(spa_c_p.as_ref().err())
.or(sd.as_ref().err())
.map(ToString::to_string);
if let Some(error) = field_error {
for cs in scores.iter_mut() {
cs.field_significance_benchmark = format!("unscored: {error}");
}
} else {
let (rc_p, spa_p, spa_c_p) = (rc_p.unwrap(), spa_p.unwrap(), spa_c_p.unwrap());
for cs in scores.iter_mut() {
cs.field_reality_check_p = rc_p;
cs.field_spa_p = spa_p;
cs.field_spa_consistent_p = spa_c_p;
}
for (cs, s) in scores.iter_mut().zip(sd.unwrap()) {
cs.step_down_significant = s;
}
}
}
if min_len >= 2 && pooled.len() >= 2 {
let aligned: Vec<&[f64]> = pooled.iter().map(|p| &p[..min_len]).collect();
for (idx, cs) in scores.iter_mut().enumerate() {
let peers: Vec<&[f64]> = aligned
.iter()
.enumerate()
.filter(|&(j, _)| j != idx)
.map(|(_, &p)| p)
.collect();
let crowdedness = crate::correlation::crowdedness(aligned[idx], &peers);
cs.field_crowdedness = crowdedness.mean_corr;
cs.field_crowdedness_peers = crowdedness.n_peers;
}
}
let member: Vec<bool> = tracks.iter().map(|t| !has_no_sharpe_ratio(t)).collect();
let pareto: Vec<bool> = (0..scores.len())
.map(|i| {
member[i]
&& !(0..scores.len())
.any(|j| j != i && !pooled[j].is_empty() && dominates(&scores[j], &scores[i]))
})
.collect();
for (cs, p) in scores.iter_mut().zip(pareto) {
cs.pareto_optimal = p;
}
for cs in &mut scores {
cs.trials_sr_std_most_influential_vote = most_influential_vote.clone();
}
let sort_key = |s: &CompositeScore| match cfg.rank_key {
RankKey::DeflatedSharpe => s.composite,
RankKey::Alpha => {
if s.rank_eligible {
s.alpha
} else {
f64::NEG_INFINITY
}
}
};
scores.sort_by(|a, b| {
b.rank_eligible
.cmp(&a.rank_eligible)
.then(
sort_key(b)
.partial_cmp(&sort_key(a))
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
b.raw_mean_return
.partial_cmp(&a.raw_mean_return)
.unwrap_or(std::cmp::Ordering::Equal),
)
});
let mut group = 0usize;
let mut prev: Option<usize> = None;
for i in 0..scores.len() {
if !scores[i].rank_eligible {
continue;
}
let same_band = matches!(prev, Some(p) if ci_overlap(&scores[p], &scores[i]));
if !same_band {
group += 1;
}
scores[i].tie_group = group;
prev = Some(i);
}
let mut i = 0;
while i < scores.len() {
if !scores[i].rank_eligible {
i += 1;
continue;
}
let band = scores[i].tie_group;
let mut j = i + 1;
while j < scores.len() && scores[j].rank_eligible && scores[j].tie_group == band {
j += 1;
}
scores[i..j].sort_by(|a, b| {
b.process_score
.partial_cmp(&a.process_score)
.unwrap_or(std::cmp::Ordering::Equal)
});
i = j;
}
let mut ord = 0usize;
for cs in scores.iter_mut() {
if cs.rank_eligible {
ord += 1;
cs.rank_ordinal = ord;
}
}
let mut band_counts = vec![0usize; group + 1];
for cs in &scores {
if cs.rank_eligible {
band_counts[cs.tie_group] += 1;
}
}
for cs in scores.iter_mut() {
if cs.rank_eligible {
cs.dsr_tied = band_counts[cs.tie_group] > 1;
}
}
let mut classes: Vec<(&MandateVerdict, usize)> = scores
.iter()
.enumerate()
.filter(|(_, s)| s.declared_mandate_eligible == Some(true))
.filter_map(|(i, s)| s.verdict_applied.as_ref().map(|v| (v, i)))
.collect();
classes.sort_by(|(va, ia), (vb, ib)| {
let (a, b) = (&scores[*ia], &scores[*ib]);
class_key(va)
.cmp(&class_key(vb))
.then(
b.deflated_sharpe
.partial_cmp(&a.deflated_sharpe)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(a.agent_id.cmp(&b.agent_id))
});
let mut ordinals: Vec<(usize, usize)> = Vec::with_capacity(classes.len());
let mut prev: Option<&MandateVerdict> = None;
let mut ord = 0usize;
for (v, i) in classes {
if prev != Some(v) {
ord = 0;
}
ord += 1;
ordinals.push((i, ord));
prev = Some(v);
}
for (i, ord) in ordinals {
scores[i].declared_mandate_ordinal = Some(ord);
}
scores
}
fn ci_overlap(a: &CompositeScore, b: &CompositeScore) -> bool {
match (a.dsr_ci_low, a.dsr_ci_high, b.dsr_ci_low, b.dsr_ci_high) {
(Some(a_low), Some(a_high), Some(b_low), Some(b_high)) => {
a_low <= b_high && b_low <= a_high
}
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::deflated_sharpe::{
deflated_sharpe_ratio, expected_max_sharpe, probabilistic_sharpe_ratio, sharpe_ratio,
};
use crate::process::ProcessEvent;
fn run(mean_ret: f64, amp: f64, n: usize) -> Run {
let returns = (0..n)
.map(|i| mean_ret + amp * (i as f64 * 0.7).sin())
.collect();
Run {
returns,
trace: Trace::default(),
confidences: Vec::new(),
outcomes: Vec::new(),
cost: 0.0,
}
}
fn agent(id: &str, runs: Vec<Run>) -> AgentSubmission {
AgentSubmission {
agent_id: id.to_string(),
runs,
in_sample_trials: 0,
candidates: Vec::new(),
}
}
#[test]
fn an_invalid_dispersion_disqualifies_instead_of_inflating_the_score() {
for bad in [-1.0, f64::NAN, f64::INFINITY] {
let cfg = ScoreConfig {
trials_sr_std: bad,
..ScoreConfig::default()
};
let s = score_agent(
&agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&cfg,
);
assert!(
s.deflation_error.is_some(),
"trials_sr_std {bad} must be reported"
);
assert_eq!(s.deflated_sharpe, 0.0);
assert_eq!(s.deflation_bar_per_period, 0.0);
assert_eq!(s.dsr_ci_low, None);
assert_eq!(s.dsr_ci_high, None);
assert_eq!(s.dsr_se, None);
assert_eq!(s.composite, 0.0);
assert!(!s.rank_eligible, "an unscored agent cannot be ranked");
}
}
#[test]
fn an_invalid_frequency_disqualifies_on_the_score_and_rank_paths() {
let skilled = || {
let mut sub = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
sub.candidates = vec![
(0..30).map(|i| 0.001 + 0.0005 * (i % 2) as f64).collect(),
(0..30).map(|i| 0.001 * (i % 3) as f64).collect(),
];
sub
};
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
let runs = (0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect();
agent(&format!("a{i}"), runs)
})
.collect();
let valid = rank(&field, &ScoreConfig::default());
assert!(valid
.iter()
.all(|s| s.trials_sr_std_source != TrialsSrStdSource::Configured));
for bad in [f64::INFINITY, 0.0, -252.0, f64::NAN] {
let cfg = ScoreConfig {
periods_per_year: bad,
..ScoreConfig::default()
};
let mut scores = vec![score_agent(&skilled(), &cfg)];
scores.extend(rank(&field, &cfg));
for s in &scores {
assert_eq!(
s.deflation_error.as_deref(),
Some("periods_per_year must be finite and positive"),
"periods_per_year {bad}, {}",
s.agent_id
);
assert_eq!(s.deflated_sharpe, 0.0);
assert_eq!(s.deflation_bar_per_period, 0.0);
assert_eq!(s.dsr_ci_low, None);
assert_eq!(s.dsr_ci_high, None);
assert_eq!(s.dsr_se, None);
assert_eq!(s.composite, 0.0);
assert!(!s.rank_eligible, "periods_per_year {bad}");
}
assert_eq!(
scores[0].selection_error.as_deref(),
Some("periods_per_year must be finite and positive")
);
assert!(scores[0].selection_gap.is_none());
}
let s = score_agent(&skilled(), &ScoreConfig::default());
assert!(s.deflation_error.is_none() && s.selection_error.is_none());
assert!(s.rank_eligible);
}
#[test]
fn an_unscorable_candidate_set_reports_no_selection_gap() {
let mut sub = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
sub.candidates = vec![vec![0.001, f64::NAN, 0.002], vec![0.001; 30]];
let s = score_agent(&sub, &ScoreConfig::default());
assert_eq!(
s.selection_error.as_deref(),
Some("observation 1 must be finite")
);
assert!(s.selection_median_dsr.is_none());
assert!(s.selection_gap.is_none());
assert!(s.deflation_error.is_none());
}
#[test]
fn valid_scoring_inputs_return_the_same_numbers() {
let cfg = ScoreConfig::default();
let sub = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let s = score_agent(&sub, &cfg);
assert!(s.deflation_error.is_none() && s.selection_error.is_none());
assert!(s.rank_eligible);
let pooled = pooled_of(&sub);
let sr_std = per_period_sr_std(&cfg);
assert_eq!(
s.deflated_sharpe.to_bits(),
deflated_sharpe_ratio(&pooled, cfg.n_trials, sr_std)
.unwrap()
.to_bits()
);
assert_eq!(
s.deflation_bar_per_period.to_bits(),
expected_max_sharpe(sr_std, cfg.n_trials).unwrap().to_bits()
);
assert!(
s.dsr_ci_low.unwrap() <= s.deflated_sharpe
&& s.deflated_sharpe <= s.dsr_ci_high.unwrap()
);
}
#[test]
fn skilled_is_eligible() {
let s = score_agent(
&agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rank_eligible, "skilled should be eligible: {s:?}");
assert!(s.passed_k && s.process_ok);
}
#[test]
fn lucky_high_return_fails_pass_k() {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
let s = score_agent(&agent("lucky", runs), &ScoreConfig::default());
assert!(!s.passed_k, "lucky should fail pass^k");
assert!(!s.rank_eligible, "lucky must not be rank-eligible: {s:?}");
}
#[test]
fn process_violator_is_disqualified() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("violator", runs), &ScoreConfig::default());
assert!(!s.process_ok);
assert!(!s.rank_eligible, "a risk-gate bypass must disqualify");
}
#[test]
fn shared_support_cannot_erase_a_process_violation() {
let cfg = ScoreConfig::default();
let clean = agent("entrant", vec![run(0.002, 0.0005, 60); 2]);
let peer = agent("peer", vec![clean.runs[0].clone()]);
let clean_board = rank(&[clean.clone(), peer.clone()], &cfg);
assert!(clean_board.iter().all(|s| s.rank_eligible));
let mut violator = clean;
violator.runs[1]
.trace
.events
.push(ProcessEvent::DenylistBypass);
assert!(!score_agent(&violator, &cfg).rank_eligible);
for field in [
vec![violator.clone(), peer.clone()],
vec![peer.clone(), violator.clone()],
] {
let declarations = MandateDeclarations::from([(
"entrant".to_string(),
DeclaredMandate::AbsoluteReturn,
)]);
let board = rank_declared(&field, &declarations, &cfg);
let score = board.iter().find(|s| s.agent_id == "entrant").unwrap();
assert_eq!((score.runs_submitted, score.runs_scored), (2, 1));
assert!(!score.process_ok);
assert!(!score.rank_eligible);
assert_eq!(score.declared_mandate_eligible, Some(false));
assert_eq!(score.rank_ordinal, 0);
assert_eq!(score.composite, 0.0);
assert_eq!(score.process_score, 0.0);
assert!(score.process_floored);
assert_eq!(score.realized_floored_return, 0.0);
assert!(
board
.iter()
.find(|s| s.agent_id == "peer")
.unwrap()
.rank_eligible
);
}
}
#[test]
fn shared_support_retains_warnings_from_all_submitted_runs() {
let mut warned = agent("warned", vec![run(0.002, 0.0005, 60); 2]);
warned.runs[1]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let peer = agent("peer", vec![warned.runs[0].clone()]);
let board = rank(&[warned, peer], &ScoreConfig::default());
let score = board.iter().find(|s| s.agent_id == "warned").unwrap();
assert_eq!((score.runs_submitted, score.runs_scored), (2, 1));
assert!(score.process_ok && score.rank_eligible);
assert_eq!(score.process_warnings, 1);
assert!((score.process_score - 0.9).abs() < 1e-12);
}
#[test]
fn unavailable_bootstrap_is_reported_and_cannot_admit_an_agent() {
let entrant = agent("entrant", vec![run(0.002, 0.0005, 60); 2]);
assert!(score_agent(&entrant, &ScoreConfig::default()).rank_eligible);
let cfg = ScoreConfig {
n_boot: 0,
alpha: 2.0,
..ScoreConfig::default()
};
let score = score_agent_declared(&entrant, Some(&DeclaredMandate::AbsoluteReturn), &cfg);
assert_eq!(
score.bootstrap_error.as_deref(),
Some("n_boot must be positive")
);
assert_eq!(score.bootstrap_p, 1.0);
assert!(!score.rank_eligible);
assert_eq!(score.declared_mandate_eligible, Some(false));
}
#[test]
fn an_unsupported_dsr_interval_is_absent_from_the_report() {
let entrant = agent("entrant", vec![run(0.002, 0.0005, 60); 2]);
let cfg = ScoreConfig {
n_boot: 0,
..ScoreConfig::default()
};
let score = score_agent(&entrant, &cfg);
assert_eq!(
score.deflation_error.as_deref(),
Some("n_boot must be positive")
);
assert_eq!(
(score.dsr_ci_low, score.dsr_ci_high, score.dsr_se),
(None, None, None)
);
assert_eq!(
score.bootstrap_error.as_deref(),
Some("n_boot must be positive")
);
assert!(!score.rank_eligible);
let report = serde_json::to_value(&score).expect("serialize the score");
for field in ["dsr_ci_low", "dsr_ci_high", "dsr_se"] {
assert!(
report.get(field).is_none(),
"{field} must be absent from the report, got {:?}",
report.get(field)
);
}
assert_eq!(
report["deflation_error"], "n_boot must be positive",
"the report states why the interval is unavailable"
);
let supported = score_agent(&entrant, &ScoreConfig::default());
let report = serde_json::to_value(&supported).expect("serialize the score");
for field in ["dsr_ci_low", "dsr_ci_high", "dsr_se"] {
assert!(
report[field].is_f64(),
"{field} must be reported when estimated"
);
}
}
const CONSTANT_TRACK: &str =
"returns must not be constant: a constant series has no Sharpe ratio";
fn offsetting_seeds() -> AgentSubmission {
let (c, d) = (1.0 / 1024.0, 1.0 / 4096.0);
let seed = |sign: f64| Run {
returns: (0..60)
.map(|t| c + sign * if t % 2 == 0 { d } else { -d })
.collect(),
..Run::default()
};
agent(
"offsetting",
(0..5).flat_map(|_| [seed(1.0), seed(-1.0)]).collect(),
)
}
#[test]
fn a_constant_pooled_track_is_unavailable_and_says_why() {
let sub = offsetting_seeds();
let cfg = ScoreConfig {
execution_seeds_per_window: 2,
dsr_bar: 0.0,
..ScoreConfig::default()
};
assert!(pooled_returns(&sub, 2).iter().all(|&r| r == 1.0 / 1024.0));
let thresholds = crate::disqualification::DisqualThresholds::from_score_config(&cfg);
for score in [
score_agent(&sub, &cfg),
rank(std::slice::from_ref(&sub), &cfg).remove(0),
] {
assert_eq!(score.deflation_error.as_deref(), Some(CONSTANT_TRACK));
assert_eq!(score.bootstrap_error, None);
assert!(score.passed_k && score.process_ok && score.mandate_ok);
assert!(score.bootstrap_p < cfg.alpha);
assert_eq!((score.deflated_sharpe, score.psr), (0.0, 0.0));
assert_eq!(
(score.dsr_ci_low, score.dsr_ci_high, score.dsr_se),
(None, None, None)
);
assert!(!score.rank_eligible);
assert_eq!(
crate::disqualification::classify_disqualification(&score, &thresholds, None, None),
vec![crate::disqualification::FailReason::DeflationUnavailable]
);
}
}
#[test]
fn constant_runs_report_the_refusal_instead_of_a_score() {
let cfg = ScoreConfig::default();
for value in [0.0, 0.001] {
let runs = (0..6)
.map(|_| Run {
returns: vec![value; 60],
..Run::default()
})
.collect();
let sub = agent("flat", runs);
let score = score_agent(&sub, &cfg);
assert_eq!(
score.deflation_error.as_deref(),
Some(CONSTANT_TRACK),
"{value}"
);
assert_eq!((score.deflated_sharpe, score.psr), (0.0, 0.0));
assert_eq!(per_run_passes(&sub, None, &cfg), vec![false; 6]);
assert!(!score.passed_k && !score.rank_eligible);
let report = serde_json::to_value(&score).expect("serialize the score");
assert_eq!(report["deflation_error"], CONSTANT_TRACK);
}
}
const NON_FINITE_SHARPE: &str = "Sharpe ratio is not finite";
fn exercised(track: impl Fn(usize) -> f64) -> AgentSubmission {
let mut sub = agent(
"exercised",
(0..6)
.map(|_| Run {
returns: (0..60).map(&track).collect(),
cost: 0.25,
..Run::default()
})
.collect(),
);
sub.candidates = vec![
(0..30).map(|i| 0.001 * (i % 3) as f64).collect(),
(0..30)
.map(|i| 0.0004 * ((i * 7) % 5) as f64 - 0.0003)
.collect(),
];
sub
}
fn exercised_config() -> ScoreConfig {
ScoreConfig {
reference_dsr_population: vec![0.1, 0.5, 0.9],
..ScoreConfig::default()
}
}
fn no_sharpe_scores() -> Vec<(String, CompositeScore, &'static str)> {
let cfg = exercised_config();
let subs = [
("all zero", exercised(|_| 0.0), CONSTANT_TRACK),
("constant 0.001", exercised(|_| 0.001), CONSTANT_TRACK),
("constant -0.002", exercised(|_| -0.002), CONSTANT_TRACK),
(
"underflowing",
exercised(|i| (i % 2) as f64 * 1e-170),
NON_FINITE_SHARPE,
),
];
let mut scores = Vec::new();
for (label, sub, reason) in subs {
scores.push((format!("{label} scored"), score_agent(&sub, &cfg), reason));
scores.push((
format!("{label} ranked"),
rank(std::slice::from_ref(&sub), &cfg).remove(0),
reason,
));
}
scores
}
fn assert_null_in_report(score: &CompositeScore, fields: &[&str], label: &str) {
let report = serde_json::to_value(score).expect("serialize the score");
for field in fields {
assert_eq!(
report.get(*field),
Some(&serde_json::Value::Null),
"{label} {field}"
);
}
}
#[test]
fn a_track_with_no_sharpe_ratio_reports_no_deflated_sharpe_per_cost_or_percentile() {
let cfg = exercised_config();
assert!(!cfg.reference_dsr_population.is_empty());
for (label, score, reason) in no_sharpe_scores() {
assert_eq!(score.deflation_error.as_deref(), Some(reason), "{label}");
assert!(score.cost > 0.0, "{label}");
assert_eq!(
(score.dsr_per_cost, score.dsr_percentile),
(None, None),
"{label}"
);
assert_eq!(
(score.dsr_ci_low, score.dsr_ci_high, score.dsr_se),
(None, None, None),
"{label}"
);
assert_null_in_report(&score, &["dsr_per_cost", "dsr_percentile"], &label);
}
}
#[test]
fn a_track_with_no_sharpe_ratio_reports_no_rolling_sharpe_summary() {
for (label, score, reason) in no_sharpe_scores() {
assert_eq!(score.deflation_error.as_deref(), Some(reason), "{label}");
assert_eq!(score.rolling_windows, 340, "{label}");
assert_eq!(
(score.rolling_min_sharpe, score.rolling_frac_positive),
(None, None),
"{label}"
);
assert_null_in_report(
&score,
&["rolling_min_sharpe", "rolling_frac_positive"],
&label,
);
}
}
#[test]
fn a_track_with_no_sharpe_ratio_reports_no_revealed_selection_score() {
let control = score_agent(
&exercised(|i| 0.002 + 0.0005 * (i as f64 * 0.7).sin()),
&exercised_config(),
);
assert!(control.econ_rationality_score.is_some());
for (label, score, reason) in no_sharpe_scores() {
assert_eq!(score.deflation_error.as_deref(), Some(reason), "{label}");
assert_eq!(
(
score.econ_rationality_score,
score.econ_dominance_violations
),
(None, None),
"{label}"
);
assert!(score.selection_median_dsr.is_some(), "{label}");
assert_null_in_report(
&score,
&["econ_rationality_score", "econ_dominance_violations"],
&label,
);
}
}
#[test]
fn a_track_with_a_sharpe_ratio_reports_its_figures_unchanged() {
let cfg = exercised_config();
let dispersed = |i: usize| 0.002 + 0.0005 * (i as f64 * 0.7).sin();
let tracks: [(&str, AgentSubmission); 4] = [
("dispersed", exercised(dispersed)),
("low volatility", exercised(move |i| dispersed(i) * 1e-9)),
(
"tiny dispersion",
exercised(|i| 0.001 + 1e-12 * ((i % 5) as f64 - 2.0)),
),
("sparse", exercised(|i| if i == 59 { 1e-9 } else { 0.0 })),
];
for (label, sub) in tracks {
let score = score_agent(&sub, &cfg);
assert_eq!(score.deflation_error, None, "{label}");
let pooled = pooled_returns(&sub, 1);
let rolling = rolling_sharpe(&pooled, cfg.rolling_window).expect("spans windows");
let choice = elicit_revealed_selection(&sub.candidates, &pooled);
let mut ungated = score.clone();
ungated.rolling_min_sharpe = Some(rolling.min_sharpe);
ungated.rolling_frac_positive = Some(rolling.frac_positive);
ungated.dsr_per_cost = Some(score.deflated_sharpe / score.cost);
ungated.dsr_percentile = Some(percentile_of(
score.deflated_sharpe,
&cfg.reference_dsr_population,
));
ungated.econ_rationality_score = choice.clone().map(|c| rationality_score(&[c]));
ungated.econ_dominance_violations = choice.map(|c| usize::from(c.is_dominated()));
assert_eq!(
serde_json::to_string(&score).expect("serialize the score"),
serde_json::to_string(&ungated).expect("serialize the control"),
"{label}"
);
assert!(
score.rolling_min_sharpe.is_some()
&& score.dsr_per_cost.is_some()
&& score.dsr_percentile.is_some()
&& score.econ_rationality_score.is_some(),
"{label}"
);
}
}
#[test]
fn a_constant_run_fails_its_per_run_test() {
for (value, per_run_psr_bar) in [(0.001, 0.90), (0.0, 0.5)] {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs.push(Run {
returns: vec![value; 60],
..Run::default()
});
let sub = agent("one-flat-run", runs);
let cfg = ScoreConfig {
per_run_psr_bar,
..ScoreConfig::default()
};
assert_eq!(
checked_probabilistic_sharpe_ratio(&sub.runs[5].returns, 0.0)
.map_err(|e| e.to_string()),
Err(CONSTANT_TRACK.to_string())
);
assert_eq!(
per_run_passes(&sub, None, &cfg),
vec![true, true, true, true, true, false],
"{value}"
);
let score = score_agent(&sub, &cfg);
assert_eq!(score.deflation_error, None);
assert!(!score.passed_k && !score.rank_eligible);
assert_eq!(
crate::disqualification::classify_disqualification(
&score,
&crate::disqualification::DisqualThresholds::from_score_config(&cfg),
None,
None
),
vec![crate::disqualification::FailReason::FailedPassK],
"{value}"
);
}
}
#[test]
fn the_zero_excess_refusal_stays_a_failure_not_an_unavailability() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("buy-and-hold");
let bench = &field[0];
assert_eq!(per_run_passes(bench, Some(bench), &cfg), vec![false; 6]);
let board = rank_declared(
&field,
&MandateDeclarations::from([(
"buy-and-hold".to_string(),
DeclaredMandate::OutperformBuyAndHold,
)]),
&cfg,
);
let score = board.iter().find(|s| s.agent_id == "buy-and-hold").unwrap();
assert_eq!(
(
score.deflation_error.as_deref(),
score.bootstrap_error.as_deref()
),
(None, None)
);
assert!(!score.passed_k);
assert_eq!(score.declared_passed_k, Some(false));
assert_eq!(
score.mandate_verdict_label().as_deref(),
Some("ineligible under declared verdict (relative to buy-and-hold); host-board ineligible")
);
let one_bar = |id: &str, r: f64| {
agent(
id,
vec![Run {
returns: vec![r],
..Run::default()
}],
)
};
let zero_bar = ScoreConfig {
per_run_psr_bar: 0.0,
..cfg
};
assert_eq!(
per_run_passes(
&one_bar("entrant", 0.02),
Some(&one_bar("buy-and-hold", 0.01)),
&zero_bar
),
vec![false]
);
}
#[test]
fn a_constant_candidate_refuses_the_selection_diagnostic() {
let mut sub = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
sub.candidates = vec![
(0..30).map(|i| 0.001 * (i % 3) as f64).collect(),
vec![0.001; 30],
];
let s = score_agent(&sub, &ScoreConfig::default());
assert_eq!(s.selection_error.as_deref(), Some(CONSTANT_TRACK));
assert_eq!((s.selection_median_dsr, s.selection_gap), (None, None));
assert!(s.deflation_error.is_none() && s.rank_eligible);
}
#[test]
fn declared_eligibility_requires_available_statistics() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let bench = agent("bench", (0..5).map(|_| run(0.0005, 0.001, 60)).collect());
let base = ScoreConfig {
dsr_bar: 0.0,
alpha: 2.0,
..ScoreConfig::default()
};
let cases = [
(skilled.clone(), base.clone(), None, None),
(
skilled.clone(),
ScoreConfig {
trials_sr_std: -1.0,
..base.clone()
},
Some("trials_sr_std must be finite and non-negative"),
None,
),
(
skilled,
ScoreConfig {
n_boot: 0,
..base.clone()
},
Some("n_boot must be positive"),
Some("n_boot must be positive"),
),
(
offsetting_seeds(),
ScoreConfig {
execution_seeds_per_window: 2,
..base
},
Some(CONSTANT_TRACK),
None,
),
];
let absolute = [
DeclaredMandate::AbsoluteReturn,
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.2,
},
];
for (sub, cfg, deflation_error, bootstrap_error) in cases {
let available = deflation_error.is_none() && bootstrap_error.is_none();
let mut scores = Vec::new();
for mandate in &absolute {
scores.push(score_agent_declared(&sub, Some(mandate), &cfg));
let declarations =
MandateDeclarations::from([(sub.agent_id.clone(), mandate.clone())]);
scores.extend(rank_declared(
std::slice::from_ref(&sub),
&declarations,
&cfg,
));
}
if sub.runs.len() == bench.runs.len() {
let relative = DeclaredMandate::RelativeTo {
benchmark_id: "bench".to_string(),
};
let declarations = MandateDeclarations::from([(sub.agent_id.clone(), relative)]);
let board = rank_declared(&[sub.clone(), bench.clone()], &declarations, &cfg);
scores.extend(board.into_iter().filter(|s| s.agent_id == sub.agent_id));
}
for score in scores {
let label = format!("{:?} {:?}", score.declared_mandate, deflation_error);
assert_eq!(score.deflation_error.as_deref(), deflation_error, "{label}");
assert_eq!(score.bootstrap_error.as_deref(), bootstrap_error, "{label}");
assert_eq!(score.declared_passed_k, Some(true), "{label}");
assert_eq!(score.declared_mandate_eligible, Some(available), "{label}");
assert_eq!(score.rank_eligible, available, "{label}");
}
}
}
#[test]
fn deflation_demotes_luck() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky.clone(), skilled.clone()], &ScoreConfig::default());
let lucky_raw = board
.iter()
.find(|s| s.agent_id == "lucky")
.unwrap()
.raw_mean_return;
let skilled_raw = board
.iter()
.find(|s| s.agent_id == "skilled")
.unwrap()
.raw_mean_return;
assert!(
lucky_raw > skilled_raw,
"lucky raw {lucky_raw} should exceed skilled {skilled_raw}"
);
assert_eq!(board[0].agent_id, "skilled");
assert!(board[0].rank_eligible && !board[1].rank_eligible);
}
#[test]
fn confidence_weighting_rewards_conviction() {
let win = Run {
returns: vec![0.01; 30],
trace: Trace::default(),
confidences: vec![0.9; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let lose = Run {
returns: vec![-0.005; 30],
trace: Trace::default(),
confidences: vec![0.1; 30],
outcomes: Vec::new(),
cost: 0.0,
};
let s = score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default());
assert!(
s.confidence_weighted_return > s.raw_mean_return,
"cwr {} should beat raw {}",
s.confidence_weighted_return,
s.raw_mean_return
);
}
#[test]
fn a_confidence_outside_zero_to_one_weighs_no_more_than_one_that_is_inside() {
let weighted = |best: f64, worst: f64| {
let mut win = run(0.01, 0.0005, 30);
win.confidences = vec![best; 30];
let mut lose = run(-0.005, 0.0005, 30);
lose.confidences = vec![worst; 30];
score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default())
.confidence_weighted_return
};
assert_eq!(
weighted(1e6, 0.1),
weighted(1.0, 0.1),
"a confidence above 1 must weigh exactly as much as full conviction"
);
assert_eq!(
weighted(0.9, -1e6),
weighted(0.9, 0.0),
"a confidence below 0 must weigh exactly as little as none"
);
}
#[test]
fn stating_outcomes_does_not_move_the_conviction_weight() {
let weighted = |outcomes: Vec<bool>| {
let mut win = run(0.01, 0.0005, 30);
win.confidences = vec![0.9; 30];
win.outcomes = outcomes.clone();
let mut lose = run(-0.005, 0.0005, 30);
lose.confidences = vec![0.1; 30];
lose.outcomes = outcomes;
score_agent(&agent("conv", vec![win, lose]), &ScoreConfig::default())
.confidence_weighted_return
};
assert_eq!(weighted(Vec::new()), weighted(vec![true; 30]));
assert_eq!(weighted(Vec::new()), weighted(vec![false; 30]));
}
#[test]
fn calibration_pairs_stop_at_each_run_boundary_and_report_support() {
let mut first = run(0.002, 0.0005, 30);
first.confidences = vec![0.0, 1.0];
first.outcomes = vec![false];
let mut second = run(0.002, 0.0005, 30);
second.confidences = vec![1.0];
second.outcomes = vec![true, false];
let score = score_agent(
&agent("paired", vec![first, second]),
&ScoreConfig::default(),
);
assert_eq!(score.calibration_observations, 2);
assert_eq!(
score.calibration_brier,
Some(0.0),
"unpaired tails must not cross a run boundary and become a false error"
);
}
#[test]
fn cost_efficiency_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 30);
r.cost = 4.0;
let s = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
assert_eq!(s.cost, 4.0);
assert!(s.return_per_cost.is_some());
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 30)]),
&ScoreConfig::default(),
);
assert!(free.return_per_cost.is_none());
}
#[test]
fn in_sample_search_raises_the_deflation_bar() {
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let base = score_agent(&agent("base", runs.clone()), &ScoreConfig::default());
let mut over = agent("over", runs);
over.in_sample_trials = 5000;
let s = score_agent(&over, &ScoreConfig::default());
assert_eq!(s.effective_n_trials, 5050);
assert!(
s.deflated_sharpe <= base.deflated_sharpe,
"more in-sample search must not raise DSR ({} vs {})",
s.deflated_sharpe,
base.deflated_sharpe
);
}
#[test]
fn ranked_field_size_is_an_observable_trial_floor() {
let field: Vec<_> = (0..8)
.map(|i| {
agent(
&format!("agent-{i}"),
vec![run(0.002 + f64::from(i) * 0.00001, 0.0005, 60)],
)
})
.collect();
let cfg = ScoreConfig {
n_trials: 1,
min_field_for_measured_sr_std: usize::MAX,
..ScoreConfig::default()
};
let board = rank(&field, &cfg);
assert!(board.iter().all(|score| score.effective_n_trials == 8));
let standalone = score_agent(&field[0], &cfg);
assert_eq!(standalone.effective_n_trials, 1);
}
#[test]
fn percentile_reported_only_with_reference() {
let none = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(none.dsr_percentile.is_none());
let cfg = ScoreConfig {
reference_dsr_population: vec![0.0, 0.3, 0.6, 0.9],
..ScoreConfig::default()
};
let some = score_agent(
&agent("p", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&cfg,
);
assert!(some.dsr_percentile.is_some());
}
#[test]
fn rolling_sharpe_reported_for_long_tracks() {
let s = score_agent(
&agent("roll", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(s.rolling_min_sharpe.is_some());
let fp = s.rolling_frac_positive.expect("reported");
assert!(
(fp - 1.0).abs() < 1e-12,
"steady edge → all windows positive"
);
assert_eq!(s.rolling_windows, 280, "300 - 21 + 1 backing windows");
}
#[test]
fn rolling_sharpe_none_when_track_too_short() {
let cfg = ScoreConfig {
rolling_window: 100,
..ScoreConfig::default()
};
let s = score_agent(&agent("short", vec![run(0.002, 0.0005, 30)]), &cfg);
assert!(s.rolling_min_sharpe.is_none());
assert!(s.rolling_frac_positive.is_none());
assert_eq!(s.rolling_windows, 0);
}
#[test]
fn crowdedness_reports_the_number_of_defined_peers() {
let field = [
agent("a", vec![run(0.001, 0.003, 60)]),
agent("b", vec![run(0.002, 0.004, 60)]),
agent("c", vec![run(-0.001, 0.002, 60)]),
];
let board = rank(&field, &ScoreConfig::default());
assert!(board.iter().all(|score| score.field_crowdedness.is_some()));
assert!(board.iter().all(|score| score.field_crowdedness_peers == 2));
}
#[test]
fn dsr_per_cost_reported_only_with_cost() {
let mut r = run(0.002, 0.0005, 60);
r.cost = 5.0;
let paid = score_agent(&agent("paid", vec![r]), &ScoreConfig::default());
let dpc = paid.dsr_per_cost.expect("reported with cost");
assert!((dpc - paid.deflated_sharpe / 5.0).abs() < 1e-12);
let free = score_agent(
&agent("free", vec![run(0.002, 0.0005, 60)]),
&ScoreConfig::default(),
);
assert!(free.dsr_per_cost.is_none());
}
#[test]
fn process_violation_floors_realized_return() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.02, 0.0005, 60)).collect();
runs[0].trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: false,
});
let s = score_agent(&agent("cheater", runs), &ScoreConfig::default());
assert!(s.process_floored, "block violation must set the floor flag");
assert_eq!(
s.realized_floored_return, 0.0,
"floored to no-skill baseline"
);
assert!(
s.raw_mean_return > 0.0,
"raw return is preserved un-floored"
);
assert!(!s.rank_eligible, "eligibility logic intact");
}
#[test]
fn clean_process_is_not_floored() {
let s = score_agent(
&agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
assert!(!s.process_floored);
assert_eq!(s.realized_floored_return, s.raw_mean_return);
}
#[test]
fn overlapping_dsr_cis_flag_a_tie_and_separation_gets_a_distinct_band() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let strong_a = agent("strong_a", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let strong_b = agent("strong_b", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let weak = agent("weak", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[strong_a, strong_b, weak], &cfg);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let (a, b, w) = (get("strong_a"), get("strong_b"), get("weak"));
assert!(
a.rank_eligible && b.rank_eligible && w.rank_eligible,
"all three should clear the (deliberately low) bar"
);
assert_eq!(a.tie_group, b.tie_group, "identical CIs share a band");
assert!(a.dsr_tied && b.dsr_tied, "the pair is flagged tied");
assert_ne!(
w.tie_group, a.tie_group,
"separable agent gets a distinct band"
);
assert!(!w.dsr_tied, "a distinct band is not a tie");
assert!(
w.dsr_ci_high.unwrap() < a.dsr_ci_low.unwrap(),
"weak CI upper {} should sit below strong CI lower {}",
w.dsr_ci_high.unwrap(),
a.dsr_ci_low.unwrap()
);
}
#[test]
fn rank_ordinal_is_one_based_among_eligible() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let board = rank(&[lucky, skilled], &ScoreConfig::default());
assert_eq!(board[0].rank_ordinal, 1, "leader is ordinal 1");
assert_eq!(board[1].rank_ordinal, 0, "ineligible gets ordinal 0");
}
#[test]
fn homogeneous_field_is_unchanged_by_shared_run_set() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let lucky = {
let mut runs = vec![run(0.02, 0.002, 60)];
runs.extend((0..4).map(|_| run(0.0, 0.003, 60)));
agent("lucky", runs)
};
let steady = agent("steady", (0..5).map(|_| run(0.001, 0.001, 60)).collect());
let field = [lucky, skilled, steady];
let on = rank(&field, &ScoreConfig::default());
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(on, off, "identical cells ⇒ identical board");
assert!(on
.iter()
.all(|s| s.runs_scored == 5 && s.runs_submitted == 5));
}
#[test]
fn easy_subset_entrant_is_compared_on_the_shared_cells() {
let easy = || run(0.004, 0.0005, 60);
let hard = || run(0.0, 0.004, 60);
let veteran = agent("veteran", vec![easy(), easy(), hard(), hard(), hard()]);
let entrant = agent("entrant", vec![easy(), easy()]);
let field = [veteran, entrant];
let off = rank(
&field,
&ScoreConfig {
shared_run_set: false,
..ScoreConfig::default()
},
);
assert_eq!(off[0].agent_id, "entrant");
assert!(off[0].rank_eligible && !off[1].rank_eligible);
let on = rank(&field, &ScoreConfig::default());
let get = |id: &str| on.iter().find(|s| s.agent_id == id).unwrap();
let (v, e) = (get("veteran"), get("entrant"));
assert_eq!(v.runs_scored, 2);
assert_eq!(e.runs_scored, 2);
assert_eq!(v.runs_submitted, 5);
assert_eq!(e.runs_submitted, 2);
assert_eq!(v.deflated_sharpe.to_bits(), e.deflated_sharpe.to_bits());
assert_eq!(v.rank_eligible, e.rank_eligible);
assert_eq!(v.tie_group, e.tie_group, "indistinguishable ⇒ one band");
assert!(v.dsr_tied && e.dsr_tied);
assert!(
e.rank_ordinal == 0 || v.rank_ordinal <= e.rank_ordinal,
"the entrant must not rank above the veteran"
);
}
#[test]
fn empty_submission_does_not_empty_the_shared_cells() {
let skilled = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let ghost = agent("ghost", Vec::new());
let board = rank(&[ghost, skilled], &ScoreConfig::default());
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(get("skilled").runs_scored, 5);
assert!(get("skilled").rank_eligible);
assert!(!get("ghost").rank_eligible);
}
#[test]
fn small_field_keeps_the_configured_sr_std_byte_identical() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..4)
.map(|i| {
let m = 0.001 + 0.001 * i as f64;
agent(
&format!("a{i}"),
(0..5).map(|_| run(m, 0.001, 60)).collect(),
)
})
.collect();
let board = rank(&field, &cfg);
assert_eq!(board.len(), 4);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), per_period_sr_std(&cfg).to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(cfg.trials_sr_std));
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
assert_eq!(s.deflated_sharpe.to_bits(), alone.deflated_sharpe.to_bits());
assert_eq!(
s.dsr_ci_low.map(f64::to_bits),
alone.dsr_ci_low.map(f64::to_bits)
);
}
let pinned = rank(
&field,
&ScoreConfig {
min_field_for_measured_sr_std: usize::MAX,
..cfg
},
);
assert_eq!(board, pinned);
}
#[test]
fn large_field_measures_trials_sr_std_from_the_field() {
let cfg = ScoreConfig::default();
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let board = rank(&field, &cfg);
let mut sharpes: Vec<f64> = field
.iter()
.map(|a| {
let pooled: Vec<f64> = a
.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect();
sharpe_ratio(&pooled)
})
.collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let expected = std_dev(&sharpes);
assert!(expected > 0.0 && expected.is_finite());
let mut moved = false;
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let alone = score_agent(
field.iter().find(|a| a.agent_id == s.agent_id).unwrap(),
&cfg,
);
moved |= s.deflated_sharpe.to_bits() != alone.deflated_sharpe.to_bits();
}
assert!(
moved,
"measured dispersion must actually move the deflation"
);
let mut reversed = field.clone();
reversed.reverse();
let again = rank(&reversed, &cfg);
assert_eq!(again[0].trials_sr_std.to_bits(), expected.to_bits());
}
#[test]
fn measured_dispersion_cannot_fall_below_the_precommitted_floor() {
let cfg = ScoreConfig {
dedup_clones_for_measured_sr_std: false,
min_measured_trials_sr_std: 0.5,
..ScoreConfig::default()
};
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let phase = i as f64 * 36.0;
let returns = (0..180)
.map(|t| {
0.001 + 0.0004 * (std::f64::consts::TAU * (t as f64 + phase) / 180.0).sin()
})
.collect();
agent(
&format!("low-dispersion-{i}"),
vec![Run {
returns,
..Run::default()
}],
)
})
.collect();
let board = rank(&field, &cfg);
let floor = per_period_sr_std(&cfg);
assert!(board.iter().all(|s| {
s.trials_sr_std_source == TrialsSrStdSource::MeasuredFloored
&& s.trials_sr_std.to_bits() == floor.to_bits()
}));
}
fn dispersed_field(n: usize) -> Vec<AgentSubmission> {
(0..n)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| Run {
returns: (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect(),
..Run::default()
})
.collect(),
)
})
.collect()
}
fn flat_entrant(id: &str, value: f64) -> AgentSubmission {
agent(
id,
(0..5)
.map(|_| Run {
returns: vec![value; 60],
..Run::default()
})
.collect(),
)
}
fn bar_dependent(s: &CompositeScore) -> Vec<String> {
let bits = |v: Option<f64>| format!("{:?}", v.map(f64::to_bits));
vec![
format!("{}", s.trials_sr_std.to_bits()),
format!("{:?}", s.trials_sr_std_source),
bits(s.trials_sr_std_annualized),
format!("{}", s.trials_sr_std_annualized_equivalent.to_bits()),
format!("{}", s.deflation_bar_per_period.to_bits()),
format!("{}", s.deflation_bar_annualized_equivalent.to_bits()),
format!("{}", s.deflated_sharpe.to_bits()),
format!("{}", s.composite.to_bits()),
format!("{}", s.psr.to_bits()),
bits(s.dsr_ci_low),
bits(s.dsr_ci_high),
bits(s.dsr_se),
bits(s.dsr_per_cost),
bits(s.dsr_percentile),
bits(s.selection_median_dsr),
bits(s.selection_gap),
format!("{:?}", s.deflation_error),
format!("{}", s.rank_eligible),
format!("{}", s.rank_ordinal),
]
}
fn by_id<'a>(board: &'a [CompositeScore], id: &str) -> &'a CompositeScore {
board
.iter()
.find(|s| s.agent_id == id)
.expect("agent on the board")
}
#[test]
fn an_undefined_sharpe_entrant_cannot_move_another_agents_bar() {
let cfg = ScoreConfig::default();
let field = dispersed_field(5);
let baseline = rank(&field, &cfg);
assert!(baseline
.iter()
.all(|s| s.trials_sr_std_source == TrialsSrStdSource::Measured));
assert!(baseline.iter().any(|s| s.rank_eligible));
for (label, value) in [("all zero", 0.0), ("constant nonzero", 0.001)] {
let mut with_entrant = field.clone();
with_entrant.push(flat_entrant("flat", value));
let board = rank(&with_entrant, &cfg);
assert_eq!(board.len(), 6, "{label}");
for base in &baseline {
let after = by_id(&board, &base.agent_id);
assert_eq!(
bar_dependent(after),
bar_dependent(base),
"{label} moved {}",
base.agent_id
);
}
assert!(
board
.iter()
.any(|s| s.agent_id != "flat" && s.deflated_sharpe > 0.0),
"{label} drove the whole field to a zero deflated Sharpe"
);
}
}
#[test]
fn an_excluded_entrant_is_still_scored_and_refused_on_its_own_terms() {
let cfg = ScoreConfig::default();
let mut field = dispersed_field(5);
field.push(flat_entrant("flat", 0.001));
let board = rank(&field, &cfg);
let flat = by_id(&board, "flat");
assert_eq!(flat.deflation_error.as_deref(), Some(CONSTANT_TRACK));
assert_eq!((flat.deflated_sharpe, flat.psr), (0.0, 0.0));
assert!(!flat.rank_eligible && flat.runs_scored == 5);
assert_eq!(
flat.trials_sr_std.to_bits(),
by_id(&board, "a0").trials_sr_std.to_bits(),
"an excluded agent is deflated against the same field bar as everyone else"
);
}
#[test]
fn a_field_below_the_floor_after_exclusion_falls_back_to_the_configured_prior() {
let cfg = ScoreConfig::default();
let mut field = dispersed_field(4);
field.push(flat_entrant("flat-zero", 0.0));
field.push(flat_entrant("flat-nonzero", 0.001));
let board = rank(&field, &cfg);
assert_eq!(board.len(), 6);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), per_period_sr_std(&cfg).to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(cfg.trials_sr_std));
}
let lowered = rank(
&field,
&ScoreConfig {
min_field_for_measured_sr_std: 4,
..cfg.clone()
},
);
assert!(lowered
.iter()
.all(|s| s.trials_sr_std_source != TrialsSrStdSource::Configured));
}
#[test]
fn a_field_with_no_undefined_sharpe_track_measures_the_same_bytes() {
let cfg = ScoreConfig::default();
let field = dispersed_field(6);
let board = rank(&field, &cfg);
let mut sharpes: Vec<f64> = field
.iter()
.map(|a| sharpe_ratio(&pooled_returns(a, cfg.execution_seeds_per_window)))
.collect();
assert!(sharpes.iter().all(|sr| sr.is_finite()));
sharpes.sort_by(|a, b| a.partial_cmp(b).expect("finite"));
let expected = std_dev(&sharpes);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.deflation_error, None);
}
}
#[test]
fn a_constant_entrant_is_not_a_crowdedness_peer() {
let cfg = ScoreConfig::default();
let field = dispersed_field(5);
let baseline = rank(&field, &cfg);
let mut with_entrant = field.clone();
with_entrant.push(flat_entrant("flat", 0.001));
let board = rank(&with_entrant, &cfg);
for base in &baseline {
let after = by_id(&board, &base.agent_id);
assert_eq!(
after.field_crowdedness.map(f64::to_bits),
base.field_crowdedness.map(f64::to_bits),
"{}",
base.agent_id
);
assert_eq!(after.field_crowdedness_peers, base.field_crowdedness_peers);
}
let flat = by_id(&board, "flat");
assert_eq!(
(flat.field_crowdedness, flat.field_crowdedness_peers),
(None, 0)
);
}
#[test]
fn a_constant_candidate_is_not_a_revealed_selection_option() {
let cfg = ScoreConfig::default();
let dispersed_candidate: Vec<f64> = (0..30).map(|i| 0.0001 * (i % 3) as f64).collect();
let mut sub = agent("skilled", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
sub.candidates = vec![dispersed_candidate.clone(), vec![0.001; 30]];
let mut control = sub.clone();
control.candidates = vec![dispersed_candidate];
let scored = score_agent(&sub, &cfg);
let control = score_agent(&control, &cfg);
assert_eq!(scored.econ_dominance_violations, Some(0));
assert_eq!(
scored.econ_rationality_score.map(f64::to_bits),
control.econ_rationality_score.map(f64::to_bits)
);
assert_eq!(
scored.econ_dominance_violations,
control.econ_dominance_violations
);
}
fn gaussian_like(n: usize, mean_ret: f64, sd: f64, seed: u64) -> Vec<f64> {
let mut state = seed;
let mut next = || {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1_442_695_040_888_963_407);
(state >> 11) as f64 / (1u64 << 53) as f64
};
let raw: Vec<f64> = (0..n)
.map(|_| (0..12).map(|_| next()).sum::<f64>() - 6.0)
.collect();
let (m, s) = (mean(&raw), std_dev(&raw));
raw.iter().map(|x| mean_ret + sd * (x - m) / s).collect()
}
fn pooled_of(sub: &AgentSubmission) -> Vec<f64> {
sub.runs
.iter()
.flat_map(|r| r.returns.iter().copied())
.collect()
}
#[test]
fn execution_replicates_are_averaged_not_counted_as_extra_market_time() {
let sub = agent(
"replicated",
vec![
run(0.01, 0.0, 3),
run(0.03, 0.0, 3),
run(-0.02, 0.0, 3),
run(0.00, 0.0, 3),
],
);
let cfg = ScoreConfig {
execution_seeds_per_window: 2,
..ScoreConfig::default()
};
let pooled = pooled_returns(&sub, cfg.execution_seeds_per_window);
assert_eq!(pooled.len(), 6);
assert_eq!(pooled, vec![0.02, 0.02, 0.02, -0.01, -0.01, -0.01]);
let score = score_agent(&sub, &cfg);
assert_eq!(score.pooled_observations, 6);
assert_eq!(
score.psr.to_bits(),
probabilistic_sharpe_ratio(&pooled, 0.0).to_bits(),
"PSR must see the de-duplicated time axis"
);
}
#[test]
fn one_execution_per_window_preserves_legacy_pooling() {
let sub = agent("single", vec![run(0.01, 0.002, 4), run(-0.01, 0.001, 5)]);
assert_eq!(
pooled_returns(&sub, 1),
pooled_of(&sub),
"the default is byte-compatible for one execution per window"
);
}
#[test]
#[should_panic(expected = "cannot form complete")]
fn incomplete_execution_replicate_block_is_rejected() {
let sub = agent(
"bad",
vec![
run(0.01, 0.001, 3),
run(0.01, 0.001, 3),
run(0.01, 0.001, 3),
],
);
let _ = pooled_returns(&sub, 2);
}
#[test]
#[should_panic(expected = "equal return lengths")]
fn unequal_execution_replicate_lengths_are_rejected() {
let sub = agent("bad", vec![run(0.01, 0.001, 3), run(0.01, 0.001, 4)]);
let _ = pooled_returns(&sub, 2);
}
#[test]
fn annualized_prior_is_converted_per_period_once() {
let cfg = ScoreConfig {
trials_sr_std: 0.5,
periods_per_year: 8760.0,
..ScoreConfig::default()
};
let expected = 0.5 / 8760f64.sqrt();
assert_eq!(per_period_sr_std(&cfg).to_bits(), expected.to_bits());
let sub = agent("a", (0..5).map(|_| run(0.0002, 0.003, 300)).collect());
let s = score_agent(&sub, &cfg);
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Configured);
assert_eq!(s.trials_sr_std.to_bits(), expected.to_bits());
assert_eq!(s.trials_sr_std_annualized, Some(0.5));
let pooled = pooled_of(&sub);
let once = deflated_sharpe_ratio(&pooled, cfg.n_trials, expected).unwrap();
let raw = deflated_sharpe_ratio(&pooled, cfg.n_trials, 0.5).unwrap();
let twice =
deflated_sharpe_ratio(&pooled, cfg.n_trials, expected / 8760f64.sqrt()).unwrap();
assert_eq!(s.deflated_sharpe.to_bits(), once.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), raw.to_bits());
assert_ne!(s.deflated_sharpe.to_bits(), twice.to_bits());
}
#[test]
fn clone_clusters_vote_once_on_the_measured_sr_std() {
let cfg = ScoreConfig::default();
let distinct: Vec<AgentSubmission> = (0..5)
.map(|i| {
agent(
&format!("a{i}"),
vec![Run {
returns: gaussian_like(120, 0.0002 + 0.0003 * i as f64, 0.003, 11 + i),
trace: Trace::default(),
confidences: vec![],
outcomes: vec![],
cost: 0.0,
}],
)
})
.collect();
let base = rank(&distinct, &cfg);
assert_eq!(base[0].trials_sr_std_source, TrialsSrStdSource::Measured);
let off = rank(
&distinct,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert_eq!(base, off, "no clones: the collapse is the identity");
let mut flooded = distinct.clone();
for k in 0..20 {
let mut copy = distinct[2].clone();
copy.agent_id = format!("copy{k:02}");
let lever = 1.0 + 0.05 * k as f64;
copy.runs[0].returns.iter_mut().for_each(|r| *r *= lever);
flooded.push(copy);
}
let board = rank(&flooded, &cfg);
assert_eq!(board.len(), 25, "clones are still scored");
assert_eq!(
board[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits(),
"twenty copies of one stream are one vote"
);
let mut shuffled = flooded.clone();
shuffled.rotate_left(7);
shuffled.swap(0, 12);
let again = rank(&shuffled, &cfg);
assert_eq!(
again[0].trials_sr_std.to_bits(),
base[0].trials_sr_std.to_bits()
);
let exposed = rank(
&flooded,
&ScoreConfig {
dedup_clones_for_measured_sr_std: false,
..cfg.clone()
},
);
assert!(
exposed[0].trials_sr_std < base[0].trials_sr_std,
"without the collapse the copies shrink the dispersion"
);
}
#[test]
fn measured_sr_std_is_never_reconverted() {
let field: Vec<AgentSubmission> = (0..5)
.map(|i| {
let m = 0.0002 + 0.0003 * i as f64;
let phase = 0.6 * i as f64;
agent(
&format!("a{i}"),
(0..5)
.map(|_| {
let mut r = run(m, 0.003, 60);
r.returns = (0..60)
.map(|t| m + 0.003 * (t as f64 * 0.7 + phase).sin())
.collect();
r
})
.collect(),
)
})
.collect();
let mut sharpes: Vec<f64> = field.iter().map(|a| sharpe_ratio(&pooled_of(a))).collect();
sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap());
let raw_measured = std_dev(&sharpes);
for ppy in [1.0, 252.0, 8760.0] {
let cfg = ScoreConfig {
min_measured_trials_sr_std: 0.0,
..ScoreConfig::for_periods_per_year(ppy)
};
let board = rank(&field, &cfg);
for s in &board {
assert_eq!(s.trials_sr_std_source, TrialsSrStdSource::Measured);
assert_eq!(s.trials_sr_std.to_bits(), raw_measured.to_bits());
assert_eq!(s.trials_sr_std_annualized, None);
let pooled = pooled_of(field.iter().find(|a| a.agent_id == s.agent_id).unwrap());
let expected = deflated_sharpe_ratio(&pooled, cfg.n_trials, raw_measured).unwrap();
assert_eq!(s.deflated_sharpe.to_bits(), expected.to_bits());
}
}
}
#[test]
fn daily_buy_and_hold_clears_the_corrected_bar() {
let runs: Vec<Run> = (0..3)
.map(|i| Run {
returns: gaussian_like(2500, 0.0004, 0.011, 0x5B_A7 + i),
..Run::default()
})
.collect();
let index = agent("buy-and-hold", runs);
let cfg = ScoreConfig {
n_trials: 50,
trials_sr_std: 0.1,
periods_per_year: 252.0,
..ScoreConfig::default()
};
let s = score_agent(&index, &cfg);
assert!(s.psr > 0.99, "the index is clearly positive: {s:?}");
assert!(
s.rank_eligible,
"an index-like daily track must clear an annualized 0.1 prior: {s:?}"
);
let old_units = ScoreConfig {
periods_per_year: 1.0,
..cfg
};
let old = score_agent(&index, &old_units);
assert!(
!old.rank_eligible && old.deflated_sharpe < 0.05,
"applied per period the same prior was unreachable: {old:?}"
);
}
#[test]
fn hourly_bar_is_stricter_per_period_than_daily_for_the_same_annualized_prior() {
let daily = ScoreConfig::for_periods_per_year(252.0);
let hourly = ScoreConfig::for_periods_per_year(8760.0);
let star =
|cfg: &ScoreConfig| expected_max_sharpe(per_period_sr_std(cfg), cfg.n_trials).unwrap();
assert!(star(&hourly) > 0.0 && star(&daily) > 0.0);
assert!(
star(&hourly) < star(&daily),
"hourly sr_star {} must be below daily {}",
star(&hourly),
star(&daily)
);
let ann_h = star(&hourly) * 8760f64.sqrt();
let ann_d = star(&daily) * 252f64.sqrt();
assert!((ann_h - ann_d).abs() < 1e-12, "{ann_h} vs {ann_d}");
}
#[test]
fn default_min_annual_sharpe_is_identical_to_the_old_per_run_test() {
let mut runs = vec![run(0.02, 0.002, 60), run(0.0, 0.003, 60)];
runs.extend((0..3).map(|_| run(0.002, 0.0005, 60)));
let sub = agent("mixed", runs);
let clean = agent("clean", (0..3).map(|_| run(0.002, 0.0005, 60)).collect());
for ppy in [52.0, 252.0, 8760.0] {
let cfg = ScoreConfig::for_periods_per_year(ppy);
assert_eq!(per_run_psr_benchmark(&cfg).to_bits(), 0f64.to_bits());
let old: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, 0.0) >= cfg.per_run_psr_bar)
.collect();
assert!(old.iter().any(|&p| p) && old.iter().any(|&p| !p));
assert_eq!(
score_agent(&sub, &cfg).passed_k,
pass_k(&old, PassMode::All)
);
assert!(score_agent(&clean, &cfg).passed_k);
}
let strict = ScoreConfig {
per_run_min_annual_sharpe: 3.0,
..ScoreConfig::for_periods_per_year(252.0)
};
assert_eq!(
per_run_psr_benchmark(&strict).to_bits(),
(3.0 / 252f64.sqrt()).to_bits()
);
let weak = agent("weak", (0..3).map(|_| run(0.0005, 0.004, 60)).collect());
assert!(score_agent(&weak, &ScoreConfig::for_periods_per_year(252.0)).passed_k);
assert!(
!score_agent(&weak, &strict).passed_k,
"a minimum annualized Sharpe of 3.0 must fail a 0.0005/0.004 track"
);
}
fn field_with_one_single_run_failure() -> Vec<AgentSubmission> {
let clean = |id: &str| agent(id, (0..6).map(|_| run(0.002, 0.0005, 60)).collect());
let mut runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
runs.push(run(-0.002, 0.0005, 60));
vec![clean("a"), agent("one_bad_run", runs), clean("b")]
}
#[test]
fn default_pass_mode_is_all_and_matches_the_old_gate() {
let cfg = ScoreConfig::default();
assert_eq!(cfg.pass_mode, PassMode::All);
assert_eq!(cfg.mandate.max_run_drawdown, 1.0);
let field = field_with_one_single_run_failure();
let board = rank(&field, &cfg);
let benchmark = per_run_psr_benchmark(&cfg);
for s in &board {
let sub = field.iter().find(|a| a.agent_id == s.agent_id).unwrap();
let old_per_run: Vec<bool> = sub
.runs
.iter()
.map(|r| probabilistic_sharpe_ratio(&r.returns, benchmark) >= cfg.per_run_psr_bar)
.collect();
let old_passed_k = pass_k(&old_per_run, PassMode::All);
assert_eq!(s.passed_k, old_passed_k, "{}", s.agent_id);
let old_eligible = s.deflated_sharpe >= cfg.dsr_bar
&& old_passed_k
&& s.process_ok
&& s.bootstrap_p < cfg.alpha
&& s.max_drawdown <= cfg.mandate.max_drawdown;
assert_eq!(s.rank_eligible, old_eligible, "{}", s.agent_id);
}
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert!(get("a").rank_eligible && get("b").rank_eligible);
assert!(!get("one_bad_run").rank_eligible);
assert!(!get("one_bad_run").passed_k);
}
#[test]
fn any_mode_admits_a_regime_dependent_edge_the_all_mode_rejects() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.003, 0.0005, 60)).collect();
runs.push(run(-0.001, 0.0005, 60));
let sub = agent("regime_edge", runs);
let all = score_agent(&sub, &ScoreConfig::default());
assert!(!all.passed_k && !all.rank_eligible, "{all:?}");
let any = score_agent(&sub, &ScoreConfig::reliability_never_catastrophic(0.20));
assert!(any.deflated_sharpe >= 0.95);
assert!(any.bootstrap_p < 0.05 && any.process_ok && any.mandate_ok);
assert!(any.worst_run_drawdown < 0.20 && any.max_drawdown < 0.20);
assert!(any.passed_k && any.rank_eligible, "{any:?}");
}
#[test]
fn per_run_drawdown_bound_rejects_one_catastrophic_run_that_the_pooled_bound_misses() {
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
for r in &mut crash.returns[10..15] {
*r = -0.03;
}
runs.push(crash);
let sub = agent("one_blowup", runs);
let loose_pooled = ScoreConfig {
mandate: Mandate {
max_drawdown: 0.20,
max_run_drawdown: 1.0,
},
pass_mode: PassMode::Any,
..ScoreConfig::default()
};
let pooled_only = score_agent(&sub, &loose_pooled);
assert!(pooled_only.max_drawdown <= 0.20, "{pooled_only:?}");
assert!(
pooled_only.mandate_ok && pooled_only.rank_eligible,
"{pooled_only:?}"
);
let mut preset = ScoreConfig::reliability_never_catastrophic(0.10);
preset.mandate.max_drawdown = 0.20;
let bounded = score_agent(&sub, &preset);
assert!(bounded.worst_run_drawdown > 0.10, "{bounded:?}");
assert!(bounded.max_drawdown <= 0.20, "the pooled bound still holds");
assert!(bounded.passed_k, "pass^k is not what rejects it");
assert!(!bounded.mandate_ok && !bounded.rank_eligible, "{bounded:?}");
}
#[test]
fn never_catastrophic_preset_is_weaker_than_the_default() {
let mut field = field_with_one_single_run_failure();
field.pop();
let mut runs: Vec<Run> = (0..5).map(|_| run(0.004, 0.0005, 60)).collect();
let mut crash = run(0.004, 0.0005, 60);
crash.returns[10] = -0.30;
runs.push(crash);
field.push(agent("blowup", runs));
let mut lucky = vec![run(0.02, 0.002, 60)];
lucky.extend((0..5).map(|_| run(0.0, 0.003, 60)));
field.push(agent("lucky", lucky));
assert!(field.iter().all(|a| a.runs.len() == 6));
assert!(field.len() < ScoreConfig::default().min_field_for_measured_sr_std);
let default = rank(&field, &ScoreConfig::default());
let preset = rank(&field, &ScoreConfig::reliability_never_catastrophic(0.20));
let eligible = |board: &[CompositeScore]| -> Vec<String> {
let mut v: Vec<String> = board
.iter()
.filter(|s| s.rank_eligible)
.map(|s| s.agent_id.clone())
.collect();
v.sort();
v
};
let (d, p) = (eligible(&default), eligible(&preset));
assert_eq!(d, vec!["a".to_string()], "{default:?}");
assert!(
d.iter().all(|id| p.contains(id)),
"default {d:?} not within preset {p:?}"
);
assert!(p.contains(&"one_bad_run".to_string()) && !d.contains(&"one_bad_run".to_string()));
assert!(
!p.contains(&"blowup".to_string()),
"a blow-up is refused under both"
);
assert!(
p.contains(&"lucky".to_string()) && !d.contains(&"lucky".to_string()),
"{preset:?}"
);
}
fn relative_field() -> Vec<AgentSubmission> {
let bench: Vec<Run> = (0..5)
.map(|_| run(0.002, 0.0005, 60))
.chain(std::iter::once(run(-0.003, 0.0005, 60)))
.collect();
let beats: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for (i, x) in r.returns.iter_mut().enumerate() {
*x += 0.001 + 0.0002 * (i as f64 * 0.7).sin();
}
r
})
.collect();
let mut five_of_six = beats.clone();
for (i, x) in five_of_six[5].returns.iter_mut().enumerate() {
*x = bench[5].returns[i] - 0.001;
}
let trails: Vec<Run> = bench
.iter()
.map(|b| {
let mut r = b.clone();
for x in r.returns.iter_mut() {
*x = x.abs() * 0.5 + 0.0005;
}
r
})
.collect();
vec![
agent("buy-and-hold", bench.clone()),
agent("beats", beats),
agent("five-of-six", five_of_six),
agent("trails", trails),
agent("clone", bench),
]
}
#[test]
fn fixed_significance_benchmark_is_separate_from_default_pass_verdict() {
let field = relative_field();
let default = rank(&field, &ScoreConfig::default());
let renamed = rank(
&field,
&ScoreConfig {
benchmark_agent_id: "no-such-agent".to_string(),
..ScoreConfig::default()
},
);
assert_eq!(
default.iter().map(|s| s.rank_eligible).collect::<Vec<_>>(),
renamed.iter().map(|s| s.rank_eligible).collect::<Vec<_>>(),
"changing the significance null must not change default pass^k eligibility"
);
assert!(default
.iter()
.all(|s| s.field_significance_benchmark == "buy-and-hold"));
assert!(renamed
.iter()
.all(|s| s.field_significance_benchmark == "zero-return-cash"));
assert_eq!(
serde_json::from_str::<ScoreConfig>("{\"n_trials\":1,\"trials_sr_std\":0.5,\"dsr_bar\":0.9,\"per_run_psr_bar\":0.9,\"alpha\":0.05,\"bootstrap_seed\":1,\"n_boot\":10,\"block_prob\":0.1}")
.unwrap()
.benchmark_agent_id,
"buy-and-hold"
);
}
#[test]
fn relative_verdict_tests_excess_over_the_same_cell() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("buy-and-hold");
let default = rank(&field, &ScoreConfig::default());
let relative = rank(&field, &cfg);
let passed = |board: &[CompositeScore], id: &str| {
board.iter().find(|s| s.agent_id == id).unwrap().passed_k
};
assert!(!passed(&default, "beats") && passed(&relative, "beats"));
assert!(passed(&default, "trails") && !passed(&relative, "trails"));
assert!(!passed(&default, "buy-and-hold") && !passed(&relative, "buy-and-hold"));
assert!(!passed(&relative, "clone"));
assert!(!passed(&relative, "five-of-six"));
let bench = &field[0];
let five = per_run_passes(&field[2], Some(bench), &cfg);
assert_eq!(five, vec![true, true, true, true, true, false]);
let zero = per_run_passes(bench, Some(bench), &cfg);
assert_eq!(zero, vec![false; 6]);
let lax = ScoreConfig {
per_run_psr_bar: 0.5,
..cfg.clone()
};
assert_eq!(per_run_passes(bench, Some(bench), &lax), vec![false; 6]);
for d in &default {
let r = relative.iter().find(|s| s.agent_id == d.agent_id).unwrap();
assert_eq!(d.deflated_sharpe.to_bits(), r.deflated_sharpe.to_bits());
assert_eq!(d.psr.to_bits(), r.psr.to_bits());
assert_eq!(d.bootstrap_p.to_bits(), r.bootstrap_p.to_bits());
assert_eq!(
d.worst_run_drawdown.to_bits(),
r.worst_run_drawdown.to_bits()
);
assert_eq!(d.process_ok, r.process_ok);
}
}
#[test]
fn relative_verdict_without_a_benchmark_fails_every_run() {
let field = relative_field();
let cfg = ScoreConfig::relative_to_benchmark("absent");
assert!(rank(&field, &cfg).iter().all(|s| !s.passed_k));
let alone = score_agent(
&field[1],
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert!(!alone.passed_k);
let mut short = field[0].clone();
short.runs[2].returns.pop();
let v = per_run_passes(
&field[1],
Some(&short),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, false, true, true, true]);
let mut fewer = field[0].clone();
fewer.runs.truncate(4);
let v = per_run_passes(
&field[1],
Some(&fewer),
&ScoreConfig::relative_to_benchmark("buy-and-hold"),
);
assert_eq!(v, vec![true, true, true, true, false, false]);
}
#[test]
fn relative_preset_differs_only_in_mode_and_benchmark_id() {
let mut preset = serde_json::to_value(ScoreConfig::relative_to_benchmark("index")).unwrap();
let default = serde_json::to_value(ScoreConfig::default()).unwrap();
assert_eq!(preset["pass_mode"], "relative_to_benchmark");
assert_eq!(preset["benchmark_agent_id"], "index");
preset["pass_mode"] = default["pass_mode"].clone();
preset["benchmark_agent_id"] = default["benchmark_agent_id"].clone();
assert_eq!(preset, default);
}
#[test]
fn warn_events_lower_process_score_and_count_but_not_eligibility() {
let clean = agent("clean", (0..5).map(|_| run(0.002, 0.0005, 60)).collect());
let mut warned_runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
warned_runs[3]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
let cfg = ScoreConfig::default();
let c = score_agent(&clean, &cfg);
let w = score_agent(&warned, &cfg);
assert_eq!(c.process_warnings, 0);
assert_eq!(c.process_score, 1.0);
assert_eq!(w.process_warnings, 2);
assert!((w.process_score - 0.8).abs() < 1e-9);
assert!(c.process_ok && w.process_ok);
assert_eq!(c.rank_eligible, w.rank_eligible);
assert!(w.rank_eligible);
assert_eq!(c.deflated_sharpe.to_bits(), w.deflated_sharpe.to_bits());
assert_eq!(c.composite.to_bits(), w.composite.to_bits());
}
#[test]
fn block_violation_zeroes_graded_process_score() {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
runs[1].trace.events.push(ProcessEvent::DenylistBypass);
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
let s = score_agent(&agent("blocked", runs), &ScoreConfig::default());
assert_eq!(s.process_score, 0.0);
assert_eq!(s.process_warnings, 1, "warns are still counted");
assert!(!s.process_ok && !s.rank_eligible);
}
#[test]
fn tie_band_orders_cleaner_process_first() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let clean = agent("clean", (0..3).map(|_| run(0.01, 0.001, 60)).collect());
let mut warned_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
warned_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let warned = agent("warned", warned_runs);
for field in [
vec![clean.clone(), warned.clone()],
vec![warned.clone(), clean.clone()],
] {
let board = rank(&field, &cfg);
assert_eq!(board[0].agent_id, "clean", "cleaner process leads the band");
assert_eq!(board[1].agent_id, "warned");
assert_eq!(board[0].rank_ordinal, 1);
assert_eq!(board[1].rank_ordinal, 2);
assert!(board[0].rank_eligible && board[1].rank_eligible);
assert_eq!(board[0].tie_group, board[1].tie_group);
assert!(board[0].dsr_tied && board[1].dsr_tied);
}
}
#[test]
fn process_tie_break_never_crosses_tie_bands() {
let cfg = ScoreConfig {
n_trials: 2,
trials_sr_std: 0.01,
dsr_bar: 0.10,
per_run_psr_bar: 0.05,
alpha: 0.9,
n_boot: 600,
..ScoreConfig::default()
};
let mut strong_runs: Vec<Run> = (0..3).map(|_| run(0.01, 0.001, 60)).collect();
strong_runs[0]
.trace
.events
.push(ProcessEvent::ConcentrationBreach);
let strong_warned = agent("strong_warned", strong_runs);
let weak_clean = agent("weak_clean", (0..3).map(|_| run(0.001, 0.02, 60)).collect());
let board = rank(&[weak_clean, strong_warned], &cfg);
assert!(board.iter().all(|s| s.rank_eligible));
assert_eq!(board[0].agent_id, "strong_warned");
assert_ne!(board[0].tie_group, board[1].tie_group, "separable bands");
assert!(board[0].process_score < board[1].process_score);
}
#[test]
fn econ_rationality_reported_from_declared_candidates() {
let cfg = ScoreConfig::default();
let runs: Vec<Run> = (0..5).map(|_| run(0.002, 0.0005, 60)).collect();
let none = score_agent(&agent("plain", runs.clone()), &cfg);
assert!(none.econ_rationality_score.is_none());
assert!(none.econ_dominance_violations.is_none());
let mut dominated = agent("dominated", runs.clone());
dominated.candidates = vec![(0..300)
.map(|i| 0.01 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let d = score_agent(&dominated, &cfg);
assert_eq!(d.econ_rationality_score, Some(0.0));
assert_eq!(d.econ_dominance_violations, Some(1));
let mut rational = agent("rational", runs);
rational.candidates = vec![(0..300)
.map(|i| 0.0001 + 0.0005 * (i as f64 * 0.7).sin())
.collect()];
let r = score_agent(&rational, &cfg);
assert_eq!(r.econ_rationality_score, Some(1.0));
assert_eq!(r.econ_dominance_violations, Some(0));
assert_eq!(d.rank_eligible, r.rank_eligible);
}
#[test]
fn behavior_role_contributions_are_reported() {
let mixed = || {
let mut runs: Vec<Run> = (0..3).map(|_| run(0.002, 0.0005, 60)).collect();
for r in &mut runs {
r.trace.events.push(ProcessEvent::OrderPlaced {
risk_gate_passed: true,
});
}
runs[2].trace.events.push(ProcessEvent::ConcentrationBreach);
runs
};
let cfg = ScoreConfig {
execution_seeds_per_window: 3,
..ScoreConfig::default()
};
let s = score_agent(&agent("mixed", mixed()), &cfg);
let names: Vec<&str> = s
.role_contributions
.iter()
.map(|c| c.role.as_str())
.collect();
assert_eq!(names, vec!["clean_active", "warned"]);
assert!(s.role_contributions.iter().all(|c| c.periods == 60));
let again = score_agent(&agent("mixed", mixed()), &cfg);
assert_eq!(s.role_contributions, again.role_contributions);
let per_window = score_agent(&agent("mixed", mixed()), &ScoreConfig::default());
assert!(
per_window.role_contributions.is_empty(),
"{:?}",
per_window.role_contributions
);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert!(empty.role_contributions.is_empty());
}
#[test]
fn durability_is_dated_by_window_not_by_seed_order() {
let make = |m: f64, n: usize| run(m, 0.0005, n);
let per_window_means = [(0.008, 0.004), (0.004, 0.002), (0.002, 0.001)];
let ordered: Vec<Run> = per_window_means
.iter()
.flat_map(|(a, b)| [make(*a, 40), make(*b, 40)])
.collect();
let swapped: Vec<Run> = per_window_means
.iter()
.flat_map(|(a, b)| [make(*b, 40), make(*a, 40)])
.collect();
let cfg = ScoreConfig {
execution_seeds_per_window: 2,
..ScoreConfig::default()
};
let a = score_agent(&agent("ordered", ordered), &cfg);
let b = score_agent(&agent("swapped", swapped), &cfg);
assert!(
a.edge_half_life.is_some(),
"a decaying track has a half-life"
);
assert_eq!(a.edge_half_life, b.edge_half_life);
}
#[test]
fn durability_with_one_execution_per_window_is_the_per_run_series() {
let runs: Vec<Run> = [0.008, 0.004, 0.002, 0.001]
.iter()
.map(|m| run(*m, 0.0005, 40))
.collect();
let per_run: Vec<f64> = runs.iter().map(|r| mean(&r.returns)).collect();
let s = score_agent(&agent("decaying", runs), &ScoreConfig::default());
assert_eq!(s.edge_half_life, crate::decay::edge_half_life(&per_run));
assert!(s.edge_half_life.is_some());
}
#[test]
fn archived_scores_without_new_fields_deserialize_with_defaults() {
let s = score_agent(
&agent("archived", (0..3).map(|_| run(0.002, 0.0005, 60)).collect()),
&ScoreConfig::default(),
);
let mut v = serde_json::to_value(&s).expect("serialize");
let obj = v.as_object_mut().expect("object");
for field in [
"process_score",
"process_warnings",
"econ_rationality_score",
"econ_dominance_violations",
"role_contributions",
] {
assert!(obj.remove(field).is_some(), "{field} should be present");
}
let parsed: CompositeScore = serde_json::from_value(v).expect("archived JSON parses");
assert_eq!(parsed.process_score, 1.0);
assert_eq!(parsed.process_warnings, 0);
assert!(parsed.econ_rationality_score.is_none());
assert!(parsed.econ_dominance_violations.is_none());
assert!(parsed.role_contributions.is_empty());
}
#[test]
fn worst_run_drawdown_is_the_max_over_runs_not_the_pooled_track() {
let down = || Run {
returns: vec![-0.05; 4],
..Run::default()
};
let up = Run {
returns: vec![0.10; 10],
..Run::default()
};
let sub = agent("runs", vec![up.clone(), down(), down()]);
let s = score_agent(&sub, &ScoreConfig::default());
let expected = max_drawdown(&up.returns).max(max_drawdown(&down().returns));
assert_eq!(s.worst_run_drawdown.to_bits(), expected.to_bits());
assert!((s.worst_run_drawdown - (1.0 - 0.95f64.powi(4))).abs() < 1e-12);
assert_eq!(
s.max_drawdown.to_bits(),
max_drawdown(&pooled_of(&sub)).to_bits()
);
assert!((s.max_drawdown - (1.0 - 0.95f64.powi(8))).abs() < 1e-12);
assert!(s.max_drawdown > s.worst_run_drawdown);
let empty = score_agent(&agent("none", Vec::new()), &ScoreConfig::default());
assert_eq!(empty.worst_run_drawdown, 0.0);
}
#[test]
fn worst_run_drawdown_can_exceed_pooled_under_seed_averaging() {
let sub = agent(
"anticorrelated",
vec![
Run {
returns: vec![0.1, -0.2],
..Run::default()
},
Run {
returns: vec![-0.1, 0.2],
..Run::default()
},
],
);
let cfg = ScoreConfig {
execution_seeds_per_window: 2,
..ScoreConfig::default()
};
assert_eq!(pooled_returns(&sub, 2), vec![0.0, 0.0]);
let s = score_agent(&sub, &cfg);
assert_eq!(s.max_drawdown, 0.0, "pooled track is flat: {s:?}");
assert!(
(s.worst_run_drawdown - 0.20).abs() < 1e-12,
"seed one falls from 1.10 to 0.88: {}",
s.worst_run_drawdown
);
assert!(
s.worst_run_drawdown > s.max_drawdown,
"the per-run bound is not implied by the pooled bound: {s:?}"
);
let bounded = score_agent(
&sub,
&ScoreConfig {
mandate: Mandate {
max_drawdown: 0.10,
max_run_drawdown: 0.10,
},
..cfg.clone()
},
);
assert!(!bounded.mandate_ok);
let pooled_only = score_agent(
&sub,
&ScoreConfig {
mandate: Mandate {
max_drawdown: 0.10,
max_run_drawdown: 1.0,
},
..cfg
},
);
assert!(pooled_only.mandate_ok);
}
fn declare(pairs: &[(&str, DeclaredMandate)]) -> MandateDeclarations {
pairs
.iter()
.map(|(id, m)| (id.to_string(), m.clone()))
.collect()
}
fn without_declared(mut s: CompositeScore) -> CompositeScore {
s.declared_mandate = None;
s.verdict_applied = None;
s.declared_passed_k = None;
s.declared_mandate_eligible = None;
s.declared_mandate_ordinal = None;
s
}
#[test]
fn declaration_is_recorded_and_absent_by_default() {
let field = relative_field();
let cfg = ScoreConfig::default();
assert_eq!(
rank(&field, &cfg),
rank_declared(&field, &declare(&[]), &cfg)
);
let board = rank_declared(
&field,
&declare(&[
("beats", DeclaredMandate::OutperformBuyAndHold),
(
"trails",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.2,
},
),
]),
&cfg,
);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
assert_eq!(
get("beats").declared_mandate,
Some(DeclaredMandate::OutperformBuyAndHold)
);
assert_eq!(
get("beats").verdict_applied,
Some(MandateVerdict::RelativeTo {
benchmark_id: "buy-and-hold".to_string()
})
);
assert_eq!(
get("trails").verdict_applied,
Some(MandateVerdict::DrawdownCapped {
max_per_run_drawdown: 0.2
})
);
let json = serde_json::to_string(get("beats")).unwrap();
assert!(json.contains("\"declared_mandate\":{\"kind\":\"outperform_buy_and_hold\"}"));
assert!(json.contains(
"\"verdict_applied\":{\"kind\":\"relative_to\",\"benchmark_id\":\"buy-and-hold\"}"
));
for id in ["buy-and-hold", "five-of-six", "clone"] {
let s = get(id);
assert!(s.declared_mandate.is_none() && s.verdict_applied.is_none());
assert!(s.declared_passed_k.is_none() && s.declared_mandate_eligible.is_none());
let json = serde_json::to_string(s).unwrap();
assert!(!json.contains("declared_") && !json.contains("verdict_applied"));
}
assert_eq!(
get("beats").mandate_verdict_label().as_deref(),
Some(if get("beats").declared_mandate_eligible == Some(true) {
"eligible under declared verdict (relative to buy-and-hold); host-board ineligible"
} else {
"ineligible under declared verdict (relative to buy-and-hold); host-board ineligible"
})
);
assert!(get("clone").mandate_verdict_label().is_none());
let wire = r#"[{"agent_id":"a","runs":[{"returns":[0.01,0.02]}],
"declared_mandate":{"kind":"drawdown_capped","max_per_run_drawdown":0.2}},
{"agent_id":"b","runs":[{"returns":[0.01,0.02]}]}]"#;
let parsed: Vec<DeclaredSubmission> = serde_json::from_str(wire).unwrap();
let (subs, decls) = split_declarations(parsed);
assert_eq!(subs.len(), 2);
assert_eq!(subs[0].runs[0].returns, vec![0.01, 0.02]);
assert_eq!(decls.len(), 1);
assert_eq!(
decls["a"],
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.2
}
);
}
#[test]
fn buy_and_hold_under_long_only_beta_gets_the_relative_verdict_and_fails_it() {
let field = relative_field();
let board = rank_declared(
&field,
&declare(&[
("buy-and-hold", DeclaredMandate::OutperformBuyAndHold),
("clone", DeclaredMandate::OutperformBuyAndHold),
("beats", DeclaredMandate::OutperformBuyAndHold),
]),
&ScoreConfig::default(),
);
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let relative = Some(MandateVerdict::RelativeTo {
benchmark_id: "buy-and-hold".to_string(),
});
assert_eq!(get("buy-and-hold").verdict_applied, relative);
assert_eq!(get("buy-and-hold").declared_passed_k, Some(false));
assert_eq!(get("buy-and-hold").declared_mandate_eligible, Some(false));
assert_eq!(
get("buy-and-hold").mandate_verdict_label().as_deref(),
Some(
"ineligible under declared verdict (relative to buy-and-hold); host-board ineligible"
)
);
assert_eq!(get("clone").declared_passed_k, Some(false));
assert_eq!(get("beats").declared_passed_k, Some(true));
let plain = rank(&field, &ScoreConfig::default());
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(&without_declared(s.clone()), p, "{}", s.agent_id);
}
}
#[test]
fn a_declaration_cannot_flip_dsr_eligibility_or_move_the_host_columns() {
let field = field_with_one_single_run_failure();
let kinds = [
DeclaredMandate::AbsoluteReturn,
DeclaredMandate::OutperformBuyAndHold,
DeclaredMandate::RelativeTo {
benchmark_id: "a".to_string(),
},
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
];
let cfg = ScoreConfig::default();
let plain = rank(&field, &cfg);
for kind in &kinds {
let decls: MandateDeclarations = field
.iter()
.map(|s| (s.agent_id.clone(), kind.clone()))
.collect();
let board = rank_declared(&field, &decls, &cfg);
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(&without_declared(s.clone()), p, "{kind:?} {}", s.agent_id);
if s.declared_mandate_eligible == Some(true) {
assert!(s.deflated_sharpe >= cfg.dsr_bar);
assert!(s.bootstrap_p < cfg.alpha);
assert!(s.process_ok && s.mandate_ok);
}
}
if *kind == DeclaredMandate::AbsoluteReturn {
for s in &board {
assert_eq!(s.declared_mandate_eligible, Some(s.rank_eligible));
assert_eq!(s.declared_passed_k, Some(s.passed_k));
}
}
let strict = ScoreConfig {
dsr_bar: 10.0,
..ScoreConfig::default()
};
for s in rank_declared(&field, &decls, &strict) {
assert_eq!(s.declared_mandate_eligible, Some(false), "{kind:?}");
assert!(!s.rank_eligible);
}
}
let capped = rank_declared(
&field,
&declare(&[(
"one_bad_run",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
)]),
&cfg,
);
let bad = capped.iter().find(|s| s.agent_id == "one_bad_run").unwrap();
assert!(!bad.rank_eligible && !bad.passed_k);
assert_eq!(bad.declared_passed_k, Some(true));
assert_eq!(bad.declared_mandate_eligible, Some(true), "{bad:?}");
assert_eq!(
bad.mandate_verdict_label().as_deref(),
Some(
"eligible under declared verdict (drawdown capped at 0.50 per run); host-board ineligible"
)
);
}
#[test]
fn misdeclared_mandates_fail_closed() {
let field = relative_field();
let cfg = ScoreConfig::default();
let board = rank_declared(
&field,
&declare(&[(
"beats",
DeclaredMandate::RelativeTo {
benchmark_id: "absent".to_string(),
},
)]),
&cfg,
);
let beats = board.iter().find(|s| s.agent_id == "beats").unwrap();
assert_eq!(beats.declared_passed_k, Some(false));
assert_eq!(beats.declared_mandate_eligible, Some(false));
let ok = rank_declared(
&field,
&declare(&[("beats", DeclaredMandate::OutperformBuyAndHold)]),
&cfg,
);
assert_eq!(
ok.iter()
.find(|s| s.agent_id == "beats")
.unwrap()
.declared_passed_k,
Some(true)
);
let alone = score_agent_declared(
&field[1],
Some(&DeclaredMandate::OutperformBuyAndHold),
&cfg,
);
assert_eq!(alone.declared_passed_k, None);
assert_eq!(alone.declared_mandate_eligible, None);
assert_eq!(
score_agent_declared(&field[1], None, &cfg),
score_agent(&field[1], &cfg)
);
let clean = field_with_one_single_run_failure();
let capped = |x: f64| {
let board = rank_declared(
&clean,
&declare(&[(
"a",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: x,
},
)]),
&cfg,
);
let a = board.iter().find(|s| s.agent_id == "a").unwrap().clone();
(a.declared_passed_k, a.declared_mandate_eligible)
};
assert_eq!(capped(0.5), (Some(true), Some(true)));
for bad in [0.0, -0.2, 1.5, f64::NAN, f64::INFINITY] {
assert_eq!(capped(bad), (Some(true), Some(false)), "bound {bad}");
}
}
#[test]
fn declared_eligibility_is_ranked_within_its_mandate_class_only() {
let field = field_with_one_single_run_failure();
let cfg = ScoreConfig::default();
let plain = rank(&field, &cfg);
let board = rank_declared(
&field,
&declare(&[
(
"one_bad_run",
DeclaredMandate::DrawdownCapped {
max_per_run_drawdown: 0.5,
},
),
("a", DeclaredMandate::AbsoluteReturn),
("b", DeclaredMandate::AbsoluteReturn),
]),
&cfg,
);
let order = |b: &[CompositeScore]| b.iter().map(|s| s.agent_id.clone()).collect::<Vec<_>>();
assert_eq!(order(&board), order(&plain));
let get = |id: &str| board.iter().find(|s| s.agent_id == id).unwrap();
let bad = get("one_bad_run");
assert!(!bad.rank_eligible && bad.rank_ordinal == 0);
assert_eq!(bad.declared_mandate_eligible, Some(true));
assert_eq!(bad.declared_mandate_ordinal, Some(1));
let (a, b) = (get("a"), get("b"));
assert!(a.rank_eligible && b.rank_eligible);
let ords: Vec<Option<usize>> = [a, b].iter().map(|s| s.declared_mandate_ordinal).collect();
assert!(
ords.contains(&Some(1)) && ords.contains(&Some(2)),
"{ords:?}"
);
let (first, second) = if a.declared_mandate_ordinal == Some(1) {
(a, b)
} else {
(b, a)
};
assert!(
first.deflated_sharpe > second.deflated_sharpe
|| (first.deflated_sharpe == second.deflated_sharpe
&& first.agent_id < second.agent_id)
);
for s in &board {
let p = plain.iter().find(|x| x.agent_id == s.agent_id).unwrap();
assert_eq!(s.rank_ordinal, p.rank_ordinal);
}
}
}