mod common;
mod elo;
mod engine;
mod mean_diff;
mod numeric;
mod outcome;
mod paired_elo;
mod quantile_diff;
mod relative_diff;
mod sign_test;
mod winrate;
pub(crate) use common::{DiffCollector, ROBUST_SPAN_MIN_POSITIVE_BASELINES};
pub(crate) use engine::MetricAggregator;
pub use engine::{compute, compute_many};
pub(crate) use numeric::checked_difference;
pub(crate) use numeric::validate_finite_output;
pub(crate) use outcome::{effective_outcome, tally_status};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use crate::error::VeridictError;
use crate::input::Record;
use crate::report::{PairScoreCounts, ScaleDiagnostics};
use crate::{
BootstrapMethod, CiMethod, FailurePolicy, IntoRecordResult, MetricConfig, MetricKind, Outcome,
TrialStatus,
};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct FailureCounts {
pub timeout: u64,
pub crash: u64,
pub invalid: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct FailureBreakdown {
pub baseline: FailureCounts,
pub candidate: FailureCounts,
}
pub struct MetricOutput {
pub effect: f64,
pub ci_low: f64,
pub ci_high: f64,
pub baseline_count: u64,
pub candidate_count: u64,
pub paired_count: u64,
pub timeouts: u64,
pub crashes: u64,
pub invalid: u64,
pub failures: FailureBreakdown,
pub warning: Option<String>,
pub records_with_id: u64,
pub max_id_count: u64,
pub quantile: Option<f64>,
pub cluster_count: Option<u64>,
pub max_cluster_size: Option<u64>,
pub effective_sample_size: Option<f64>,
pub design_effect: Option<f64>,
pub scale_diagnostics: Option<ScaleDiagnostics>,
pub tied_count: Option<u64>,
pub pair_score_counts: Option<PairScoreCounts>,
}
pub(crate) fn metric_label(metric: MetricKind) -> &'static str {
match metric {
MetricKind::WinRate => "metric winrate",
MetricKind::MeanDiff => "metric mean-diff",
MetricKind::SignTest => "metric sign-test",
MetricKind::Elo => "metric elo",
MetricKind::PairedElo => "metric paired-elo",
MetricKind::QuantileDiff => "metric quantile-diff",
MetricKind::RelativeDiff => "metric relative-diff",
}
}
pub(crate) fn lacks_closed_form_ci(metric: MetricKind) -> bool {
matches!(
metric,
MetricKind::MeanDiff
| MetricKind::PairedElo
| MetricKind::QuantileDiff
| MetricKind::RelativeDiff
)
}
pub(crate) fn ci_method_label(ci_method: CiMethod) -> &'static str {
match ci_method {
CiMethod::Wilson => "wilson",
CiMethod::Exact => "exact",
CiMethod::Jeffreys => "jeffreys",
}
}
pub(crate) fn failure_policy_label(policy: FailurePolicy) -> &'static str {
match policy {
FailurePolicy::ReportOnly => "report-only",
FailurePolicy::Exclude => "exclude",
FailurePolicy::Loss => "loss",
}
}
pub(crate) fn bootstrap_method_label(method: BootstrapMethod) -> &'static str {
match method {
BootstrapMethod::Percentile => "percentile",
BootstrapMethod::Bca => "bca",
BootstrapMethod::Basic => "basic",
}
}
#[cfg(test)]
mod tests;