agent_evaluate_contract/
metric.rs

1//! Metric taxonomy, report slices and metric health.
2//!
3//! Infra owns the taxonomy, its gate semantics and the statistics; the domain
4//! owns which class a given evaluator belongs to and whether to retire it.
5
6use serde::{Deserialize, Serialize};
7
8/// How a metric participates in release decisions.
9#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
10#[serde(rename_all = "snake_case")]
11pub enum MetricClass {
12    /// Quality we actively push up. Regressions gate on statistical evidence.
13    #[default]
14    Goal,
15    /// Must never break. A single failure gates without a significance test.
16    Guardrail,
17    /// Observed only. Never contributes to `EvalCaseResult::passed` or a gate.
18    Diagnostic,
19}
20
21impl MetricClass {
22    /// Diagnostic metrics are deliberately excluded from case verdicts so a
23    /// team can observe a new signal without making CI red.
24    pub const fn affects_case_verdict(self) -> bool {
25        matches!(self, Self::Goal | Self::Guardrail)
26    }
27
28    pub const fn as_str(self) -> &'static str {
29        match self {
30            Self::Goal => "goal",
31            Self::Guardrail => "guardrail",
32            Self::Diagnostic => "diagnostic",
33        }
34    }
35}
36
37/// The action a moving metric triggers. A metric with no action is noise.
38#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
39#[serde(rename_all = "snake_case")]
40pub enum MetricDecision {
41    #[default]
42    BlockRelease,
43    RollBack,
44    Investigate,
45    ObserveOnly,
46}
47
48impl MetricDecision {
49    pub const fn is_blocking(self) -> bool {
50        matches!(self, Self::BlockRelease | Self::RollBack)
51    }
52}
53
54#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
55#[serde(rename_all = "camelCase", deny_unknown_fields)]
56pub struct MetricSpec {
57    #[serde(default)]
58    pub class: MetricClass,
59    #[serde(default)]
60    pub decision: MetricDecision,
61    /// Contacted when the metric is recommended for retirement.
62    #[serde(default)]
63    pub owner: String,
64    /// Guardrails leave this unset: staying green forever is their job.
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub retire_after_idle_runs: Option<u32>,
67}
68
69impl Default for MetricSpec {
70    /// Legacy evaluators registered before the taxonomy existed keep their
71    /// blocking behaviour.
72    fn default() -> Self {
73        Self {
74            class: MetricClass::Goal,
75            decision: MetricDecision::BlockRelease,
76            owner: String::new(),
77            retire_after_idle_runs: None,
78        }
79    }
80}
81
82/// Human-label agreement accumulated for one evaluator id.
83#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
84#[serde(rename_all = "camelCase")]
85pub struct JudgeCalibration {
86    pub labeled_samples: u64,
87    pub agreed_samples: u64,
88}
89
90impl JudgeCalibration {
91    pub fn agreement(&self) -> f64 {
92        if self.labeled_samples == 0 {
93            0.0
94        } else {
95            self.agreed_samples as f64 / self.labeled_samples as f64
96        }
97    }
98}
99
100/// Minimum human-label evidence before a model-graded evaluator may gate.
101pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
102pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;
103
104/// One observed slice of a metric: successes over a bounded denominator.
105#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
106#[serde(rename_all = "camelCase")]
107pub struct RateSlice {
108    pub observed: u64,
109    pub passed: u64,
110}
111
112impl RateSlice {
113    pub fn rate(&self) -> f64 {
114        if self.observed == 0 {
115            0.0
116        } else {
117            self.passed as f64 / self.observed as f64
118        }
119    }
120}
121
122/// Aggregated outcome of one evaluator version within a single report.
123#[derive(Debug, Clone, Serialize, Deserialize)]
124#[serde(rename_all = "camelCase")]
125pub struct MetricSummary {
126    pub evaluator_id: String,
127    pub evaluator_version: String,
128    pub class: MetricClass,
129    #[serde(default)]
130    pub decision: MetricDecision,
131    pub observed: u64,
132    pub passed: u64,
133    pub rate: f64,
134    /// Wilson score interval. Suites are commonly 30-50 cases, where the
135    /// normal approximation is unusable near 0 and 1.
136    pub wilson_low: f64,
137    pub wilson_high: f64,
138    #[serde(default)]
139    pub cost_micros: u64,
140    #[serde(default)]
141    pub by_category: std::collections::BTreeMap<String, RateSlice>,
142    #[serde(default)]
143    pub by_variant: std::collections::BTreeMap<String, RateSlice>,
144}
145
146// ---------------------------------------------------------------------------
147// Candidate metrics
148// ---------------------------------------------------------------------------
149
150#[derive(Debug, Clone, Serialize, Deserialize)]
151#[serde(rename_all = "camelCase", deny_unknown_fields)]
152pub struct ListMetricCandidatesRequest {
153    #[serde(default, skip_serializing_if = "Option::is_none")]
154    pub run_id: Option<String>,
155    /// Labels seen fewer times than this are ignored as anecdotes.
156    #[serde(default = "default_min_occurrences")]
157    pub minimum_occurrences: u64,
158}
159
160fn default_min_occurrences() -> u64 {
161    3
162}
163
164/// A labelled failure mode and whether it actually predicts the outcome the
165/// system exists to decide.
166#[derive(Debug, Clone, Serialize, Deserialize)]
167#[serde(rename_all = "camelCase")]
168pub struct MetricCandidate {
169    pub label: String,
170    pub occurrences: u64,
171    pub failed_occurrences: u64,
172    /// Phi coefficient between the label and a failing human outcome.
173    pub correlation: f64,
174    /// False when the label is frequent but uncorrelated with the decision.
175    pub recommended: bool,
176    pub rationale: String,
177}
178
179#[derive(Debug, Clone, Serialize, Deserialize)]
180#[serde(rename_all = "camelCase")]
181pub struct MetricCandidateReport {
182    pub labeled_reviews: u64,
183    pub failed_reviews: u64,
184    pub candidates: Vec<MetricCandidate>,
185}
186
187// ---------------------------------------------------------------------------
188// Metric health
189// ---------------------------------------------------------------------------
190
191fn default_health_window() -> u32 {
192    100
193}
194
195#[derive(Debug, Clone, Serialize, Deserialize)]
196#[serde(rename_all = "camelCase", deny_unknown_fields)]
197pub struct MetricHealthRequest {
198    /// Most recent runs inspected per evaluator version.
199    #[serde(default = "default_health_window")]
200    pub window_runs: u32,
201    #[serde(default, skip_serializing_if = "Option::is_none")]
202    pub evaluator_id: Option<String>,
203}
204
205impl Default for MetricHealthRequest {
206    fn default() -> Self {
207        Self {
208            window_runs: default_health_window(),
209            evaluator_id: None,
210        }
211    }
212}
213
214#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
215#[serde(rename_all = "snake_case")]
216pub enum MetricRecommendation {
217    Keep,
218    Retire,
219    Investigate,
220}
221
222/// Per-run observation of one evaluator version. Written once when a run's
223/// report is sealed, so health queries never rescan case results.
224#[derive(Debug, Clone, Serialize, Deserialize)]
225#[serde(rename_all = "camelCase")]
226pub struct MetricObservation {
227    pub evaluator_id: String,
228    pub evaluator_version: String,
229    pub class: MetricClass,
230    pub run_id: String,
231    pub observed: u64,
232    pub passed: u64,
233    pub cost_micros: u64,
234    /// Variants compared in the run that produced this observation.
235    pub variant_count: u64,
236    /// True when the metric separated at least two variants of that run.
237    pub discriminated: bool,
238    pub created_at_ms: i64,
239}
240
241#[derive(Debug, Clone, Serialize, Deserialize)]
242#[serde(rename_all = "camelCase")]
243pub struct MetricHealth {
244    pub evaluator_id: String,
245    pub evaluator_version: String,
246    pub class: MetricClass,
247    pub runs: u64,
248    pub observed: u64,
249    pub passed: u64,
250    pub rate: f64,
251    /// Consecutive most-recent runs without a single failure.
252    pub idle_runs: u64,
253    /// Runs where the metric separated variants, over runs that had variants.
254    pub discrimination: f64,
255    pub discrimination_opportunities: u64,
256    pub cost_micros_per_run: u64,
257    pub labeled_samples: u64,
258    pub human_agreement: f64,
259    pub recommendation: MetricRecommendation,
260    pub rationale: String,
261}
262
263#[derive(Debug, Clone, Serialize, Deserialize)]
264#[serde(rename_all = "camelCase")]
265pub struct MetricHealthReport {
266    pub metrics: Vec<MetricHealth>,
267}
agent_evaluate_contract/metric.rs

agent_evaluate_contract/
metric.rs
