pub const MIN_DECISIVE_UNITS: u64 = 2;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Verdict {
Pass,
Fail,
Inconclusive,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum InconclusiveKind {
Noise,
Directional,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Validity {
Valid,
Invalid,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Promotion {
Promoted,
NotPromoted,
}
impl Promotion {
pub fn decide(validity: Validity, verdict: Verdict) -> Self {
if validity == Validity::Valid && verdict == Verdict::Pass {
Promotion::Promoted
} else {
Promotion::NotPromoted
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct FailureCaps {
pub max_timeouts: Option<u64>,
pub max_crashes: Option<u64>,
pub max_invalid: Option<u64>,
}
impl FailureCaps {
pub fn check(&self, timeouts: u64, crashes: u64, invalid: u64) -> (Validity, Option<String>) {
if let Some(max) = self.max_timeouts
&& timeouts > max
{
return (
Validity::Invalid,
Some(format!(
"{timeouts} timeout(s) exceeds the configured cap of {max}"
)),
);
}
if let Some(max) = self.max_crashes
&& crashes > max
{
return (
Validity::Invalid,
Some(format!(
"{crashes} crash(es) exceeds the configured cap of {max}"
)),
);
}
if let Some(max) = self.max_invalid
&& invalid > max
{
return (
Validity::Invalid,
Some(format!(
"{invalid} invalid result(s) exceeds the configured cap of {max}"
)),
);
}
(Validity::Valid, None)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrialStatus {
Ok,
Timeout,
Crash,
Invalid,
}
impl TrialStatus {
pub fn parse(s: &str) -> Option<Self> {
match s {
"ok" => Some(Self::Ok),
"timeout" => Some(Self::Timeout),
"crash" => Some(Self::Crash),
"invalid" => Some(Self::Invalid),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Outcome {
BaselineWin,
CandidateWin,
Draw,
}
impl Outcome {
pub fn parse(s: &str) -> Option<Self> {
match s {
"baseline_win" => Some(Self::BaselineWin),
"candidate_win" => Some(Self::CandidateWin),
"draw" => Some(Self::Draw),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MatchOutcome {
AWin,
BWin,
Draw,
}
impl MatchOutcome {
pub fn parse(s: &str) -> Option<Self> {
match s {
"a_win" => Some(Self::AWin),
"b_win" => Some(Self::BWin),
"draw" => Some(Self::Draw),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum MetricKind {
#[serde(rename = "winrate")]
WinRate,
#[serde(rename = "mean-diff")]
MeanDiff,
#[serde(rename = "sign-test")]
SignTest,
#[serde(rename = "elo")]
Elo,
#[serde(rename = "paired-elo")]
PairedElo,
#[serde(rename = "quantile-diff")]
QuantileDiff,
#[serde(rename = "relative-diff")]
RelativeDiff,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CiMethod {
Wilson,
Exact,
Jeffreys,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BootstrapMethod {
Percentile,
Bca,
Basic,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailurePolicy {
ReportOnly,
Exclude,
Loss,
}