rustev_eval/
repeat_config.rs

1//! Repeat configurations, `rustev.repeat-config/1` (spec 029, 3.3.1).
2//!
3//! The declaration a repeat set's statistics mean anything under: the
4//! bootstrap method, generator, level, resample count and seed; the noise
5//! band rule and floors; the reported slices; the stability metrics; and an
6//! optional top-k ranking parameter. It carries no binary64 number
7//! (fractions are Decimal strings), and its identity is the tagged digest of
8//! its canonical bytes.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use rustev_contract::decimal::Decimal;
13use rustev_contract::ids::ContentDigest;
14use rustev_contract::limits::REPLAY_V1;
15use serde::{Deserialize, Serialize};
16
17use crate::config::ConfigError;
18use crate::generator::GENERATOR;
19
20pub const REPEAT_CONFIG: &str = "rustev.repeat-config/1";
21/// The only interval method (3.4.1).
22pub const METHOD: &str = "percentile-cluster/1";
23/// Resample bounds (3.3.1).
24pub const MIN_RESAMPLES: u64 = 1_000;
25pub const MAX_RESAMPLES: u64 = 100_000;
26/// The top-k bound (3.2.4).
27pub const MAX_TOP_K: u64 = 100;
28
29#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
30#[serde(deny_unknown_fields)]
31pub struct RepeatConfig {
32    pub schema: String,
33    pub bootstrap: BootstrapSpec,
34    pub noise: Noise,
35    pub slices: Vec<SliceSpec>,
36    pub stability_metrics: Vec<String>,
37    pub top_k: Option<TopK>,
38}
39
40crate::doc::eval_document!(RepeatConfig, REPEAT_CONFIG, REPLAY_V1, ContentDigest);
41
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
43#[serde(deny_unknown_fields)]
44pub struct BootstrapSpec {
45    pub method: String,
46    pub prng: String,
47    /// In [0.5, 0.999].
48    pub level: Decimal,
49    /// 1 000 to 100 000.
50    pub resamples: u64,
51    /// The decimal digits of a `u64`.
52    pub seed: String,
53}
54
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct Noise {
58    pub rule: BandRule,
59    pub floor: Decimal,
60    pub overrides: BTreeMap<String, NoiseOverride>,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
64#[serde(deny_unknown_fields)]
65pub struct NoiseOverride {
66    pub rule: BandRule,
67    pub floor: Decimal,
68}
69
70/// `{ "k_sd": { "k": decimal } }` or `"interval_width"` (3.5.1).
71#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
72#[serde(rename_all = "snake_case")]
73pub enum BandRule {
74    KSd(KSd),
75    IntervalWidth,
76}
77
78#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
79#[serde(deny_unknown_fields)]
80pub struct KSd {
81    /// In (0, 10].
82    pub k: Decimal,
83}
84
85#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
86#[serde(deny_unknown_fields)]
87pub struct SliceSpec {
88    pub name: String,
89    /// At least 1.
90    pub min_cases: u64,
91}
92
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
94#[serde(deny_unknown_fields)]
95pub struct TopK {
96    pub param: String,
97    /// 1 to 100.
98    pub k: u64,
99}
100
101/// The decimal digits of a `u64`, in their one spelling: no sign, no
102/// leading zero (except `0` itself).
103pub fn parse_seed(s: &str) -> Option<u64> {
104    let canonical =
105        !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) && (s == "0" || !s.starts_with('0'));
106    if canonical { s.parse().ok() } else { None }
107}
108
109impl RepeatConfig {
110    /// The rule and floor for `metric`: its override when declared.
111    pub fn band_for(&self, metric: &str) -> (&BandRule, Decimal) {
112        match self.noise.overrides.get(metric) {
113            Some(o) => (&o.rule, o.floor),
114            None => (&self.noise.rule, self.noise.floor),
115        }
116    }
117
118    /// The level's Decimal units (level times 10^9), used by the exact
119    /// percentile ranks (3.4.7).
120    pub fn level_units(&self) -> i128 {
121        self.bootstrap.level.units()
122    }
123
124    /// Every rule of 3.3.1 that the document alone decides.
125    pub fn check(&self) -> Result<(), ConfigError> {
126        let bad = |m: &str| Err(ConfigError(m.into()));
127        let b = &self.bootstrap;
128        if b.method != METHOD {
129            return bad("the bootstrap method is `percentile-cluster/1`");
130        }
131        if b.prng != GENERATOR {
132            return bad("the generator is `xoshiro256starstar-splitmix64/1`");
133        }
134        if b.level < Decimal::parse("0.5").expect("literal")
135            || b.level > Decimal::parse("0.999").expect("literal")
136        {
137            return bad("the level is in [0.5, 0.999]");
138        }
139        if !(MIN_RESAMPLES..=MAX_RESAMPLES).contains(&b.resamples) {
140            return bad("resamples are 1000 to 100000");
141        }
142        if parse_seed(&b.seed).is_none() {
143            return bad("the seed is the decimal digits of a u64, without leading zeros");
144        }
145        let ten = Decimal::from_i64(10);
146        let rule_ok = |r: &BandRule| match r {
147            BandRule::KSd(KSd { k }) => *k > Decimal::ZERO && *k <= ten,
148            BandRule::IntervalWidth => true,
149        };
150        if !rule_ok(&self.noise.rule) || self.noise.overrides.values().any(|o| !rule_ok(&o.rule)) {
151            return bad("a k_sd multiplier is in (0, 10]");
152        }
153        if self.noise.floor.is_negative()
154            || self.noise.overrides.values().any(|o| o.floor.is_negative())
155        {
156            return bad("noise floors are non-negative");
157        }
158        let mut names = BTreeSet::new();
159        for s in &self.slices {
160            if s.name.is_empty() || s.name == "all" || !names.insert(s.name.as_str()) {
161                return bad("slice names are non-empty, unique, and none is `all`");
162            }
163            if s.min_cases < 1 {
164                return bad("a slice's minimum is at least 1 case");
165            }
166        }
167        let stability: BTreeSet<&str> = self.stability_metrics.iter().map(String::as_str).collect();
168        if stability.is_empty() || stability.len() != self.stability_metrics.len() {
169            return bad("stability metrics are non-empty and unique");
170        }
171        if let Some(t) = &self.top_k
172            && (t.param.is_empty() || !(1..=MAX_TOP_K).contains(&t.k))
173        {
174            return bad("top_k names a parameter and k is 1 to 100");
175        }
176        Ok(())
177    }
178
179    /// The rule that needs the evaluator configuration: every stability
180    /// metric and every override names a metric that is reported (the
181    /// configuration's metrics and series, and those 3.2.5 derives).
182    pub fn check_metrics(&self, reported: &BTreeSet<String>) -> Result<(), ConfigError> {
183        for m in self
184            .stability_metrics
185            .iter()
186            .chain(self.noise.overrides.keys())
187        {
188            if !reported.contains(m) {
189                return Err(ConfigError(format!("metric `{m}` is not reported")));
190            }
191        }
192        Ok(())
193    }
194}
rustev_eval/repeat_config.rs

rustev_eval/
repeat_config.rs
