agent_evaluate_v2_contract/
report.rs

1//! Verdicts, reports and the release gate.
2
3use std::collections::BTreeMap;
4
5use serde::{Deserialize, Serialize};
6
7use crate::catalog::{Capability, SnapshotRef};
8use crate::evaluator::{MetricClass, MetricDecision};
9use crate::run::CaseRunState;
10
11#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
12#[serde(rename_all = "camelCase")]
13pub struct AssertionVerdict {
14    pub rule: String,
15    pub passed: bool,
16    #[serde(default, skip_serializing_if = "Option::is_none")]
17    pub detail: Option<String>,
18}
19
20/// A measured value with the bound it was judged against. Values are stored
21/// even when they cannot fail: trace retention is shorter than conclusion
22/// retention, so a report can never rely on re-reading the trace later.
23#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
24#[serde(rename_all = "camelCase")]
25pub struct MetricValue {
26    pub metric_id: String,
27    pub value: f64,
28    #[serde(default, skip_serializing_if = "Option::is_none")]
29    pub min: Option<f64>,
30    #[serde(default, skip_serializing_if = "Option::is_none")]
31    pub max: Option<f64>,
32    pub passed: bool,
33    #[serde(default, skip_serializing_if = "Option::is_none")]
34    pub detail: Option<String>,
35}
36
37/// Why a judged verdict is not allowed to gate this time. Recording the
38/// reason keeps "the judge was ignored" from looking like "the judge passed".
39#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(rename_all = "snake_case")]
41pub enum VerdictDowngrade {
42    /// Inter-judge agreement fell below the evaluator's threshold.
43    LowAgreement,
44    /// A judge sharing the subject's model had its vote discarded.
45    SelfGraded,
46    /// The ensemble stopped early on its cost budget.
47    PartialEnsemble,
48    /// The judge model is not on the platform gate allowlist.
49    JudgeNotAllowlisted,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
53#[serde(rename_all = "camelCase")]
54pub struct EvaluatorVerdict {
55    pub evaluator_id: String,
56    pub evaluator_digest: String,
57    pub kind: String,
58    pub passed: bool,
59    #[serde(default)]
60    pub class: MetricClass,
61    #[serde(default)]
62    pub decision: MetricDecision,
63    #[serde(default)]
64    pub assertions: Vec<AssertionVerdict>,
65    #[serde(default)]
66    pub metrics: Vec<MetricValue>,
67    /// Judge spend, reported apart from the subject's execution cost.
68    #[serde(default)]
69    pub judge_cost_micros: u64,
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub agreement: Option<f64>,
72    #[serde(default)]
73    pub downgrades: Vec<VerdictDowngrade>,
74}
75
76impl EvaluatorVerdict {
77    /// A verdict gates only when its class allows it and nothing forced a
78    /// downgrade.
79    pub fn gates(&self) -> bool {
80        self.class.affects_case_verdict() && self.downgrades.is_empty()
81    }
82}
83
84#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
85#[serde(rename_all = "camelCase")]
86pub struct CaseResult {
87    pub case_run_id: String,
88    #[serde(rename = "snapshotRef")]
89    pub snapshot_ref: SnapshotRef,
90    pub case_id: String,
91    pub case_digest: String,
92    pub category: String,
93    pub state: CaseRunState,
94    pub passed: bool,
95    #[serde(default)]
96    pub turns_completed: usize,
97    #[serde(default)]
98    pub verdicts: Vec<EvaluatorVerdict>,
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub error: Option<String>,
101    #[serde(default)]
102    pub duration_ms: u64,
103    #[serde(default)]
104    pub cost_micros: u64,
105    #[serde(default)]
106    pub sample_index: u32,
107    #[serde(default, skip_serializing_if = "Option::is_none")]
108    pub variant_id: Option<String>,
109}
110
111/// One observed slice of a metric: successes over a bounded denominator.
112#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
113#[serde(rename_all = "camelCase")]
114pub struct RateSlice {
115    pub observed: u64,
116    pub passed: u64,
117}
118
119impl RateSlice {
120    pub fn rate(&self) -> f64 {
121        if self.observed == 0 {
122            0.0
123        } else {
124            self.passed as f64 / self.observed as f64
125        }
126    }
127}
128
129#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
130#[serde(rename_all = "camelCase")]
131pub struct MetricSummary {
132    pub evaluator_id: String,
133    pub evaluator_digest: String,
134    pub class: MetricClass,
135    #[serde(default)]
136    pub decision: MetricDecision,
137    pub observed: u64,
138    pub passed: u64,
139    pub rate: f64,
140    /// Wilson interval: suites are commonly 30-50 cases, where the normal
141    /// approximation is unusable near 0 and 1.
142    pub wilson_low: f64,
143    pub wilson_high: f64,
144    #[serde(default)]
145    pub judge_cost_micros: u64,
146    #[serde(default)]
147    pub by_category: BTreeMap<String, RateSlice>,
148    #[serde(default)]
149    pub by_variant: BTreeMap<String, RateSlice>,
150}
151
152/// Reports are revisioned: revision 1 seals once synchronous metrics are in,
153/// and each completed batch of asynchronous analysis appends a new one.
154#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
155#[serde(rename_all = "camelCase")]
156pub struct EvalReport {
157    pub id: String,
158    pub run_id: String,
159    pub revision: u32,
160    #[serde(default)]
161    pub snapshots: Vec<SnapshotRef>,
162    #[serde(default, skip_serializing_if = "Option::is_none")]
163    pub capability: Option<Capability>,
164    pub total_cases: usize,
165    pub passed_cases: usize,
166    pub quality_failed_cases: usize,
167    pub infra_failed_cases: usize,
168    pub invalid_cases: usize,
169    pub analysis_failed_cases: usize,
170    /// Pass rate over cases that produced a quality signal at all.
171    pub effective_pass_rate: f64,
172    #[serde(default)]
173    pub metrics: Vec<MetricSummary>,
174    #[serde(default)]
175    pub case_results: Vec<CaseResult>,
176    #[serde(default)]
177    pub cost_micros: u64,
178    #[serde(default)]
179    pub judge_cost_micros: u64,
180    #[serde(default)]
181    pub duration_ms: u64,
182    pub sealed_at_ms: i64,
183}
184
185#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
186#[serde(rename_all = "camelCase", deny_unknown_fields)]
187pub struct GetReportRequest {
188    pub run_id: String,
189    /// Absent means the latest sealed revision.
190    #[serde(default, skip_serializing_if = "Option::is_none")]
191    pub revision: Option<u32>,
192}
193
194#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
195#[serde(rename_all = "camelCase", deny_unknown_fields)]
196pub struct MetricThreshold {
197    pub evaluator_id: String,
198    #[serde(default, skip_serializing_if = "Option::is_none")]
199    pub minimum_rate: Option<f64>,
200    #[serde(default, skip_serializing_if = "Option::is_none")]
201    pub maximum_regression: Option<f64>,
202}
203
204/// Declarative gate: guardrails are absolute, goals are statistical, and
205/// diagnostics never gate.
206#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
207#[serde(rename_all = "camelCase", deny_unknown_fields)]
208pub struct GatePolicy {
209    #[serde(default)]
210    pub minimum_pass_rate: f64,
211    #[serde(default = "default_max_infra_rate")]
212    pub maximum_infra_failure_rate: f64,
213    #[serde(default = "default_significance")]
214    pub significance_level: f64,
215    /// Regression tolerated on a goal metric before it can block.
216    #[serde(default)]
217    pub minimum_effect: f64,
218    #[serde(default)]
219    pub goal_minimum_rate: f64,
220    #[serde(default)]
221    pub overrides: Vec<MetricThreshold>,
222}
223
224fn default_max_infra_rate() -> f64 {
225    1.0
226}
227
228fn default_significance() -> f64 {
229    0.05
230}
231
232impl Default for GatePolicy {
233    fn default() -> Self {
234        Self {
235            minimum_pass_rate: 0.0,
236            maximum_infra_failure_rate: default_max_infra_rate(),
237            significance_level: default_significance(),
238            minimum_effect: 0.0,
239            goal_minimum_rate: 0.0,
240            overrides: Vec::new(),
241        }
242    }
243}
244
245#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
246#[serde(rename_all = "camelCase", deny_unknown_fields)]
247pub struct CompareReportsRequest {
248    pub run_id: String,
249    #[serde(default, skip_serializing_if = "Option::is_none")]
250    pub revision: Option<u32>,
251    pub baseline_run_id: String,
252    #[serde(default, skip_serializing_if = "Option::is_none")]
253    pub baseline_revision: Option<u32>,
254    #[serde(default)]
255    pub policy: GatePolicy,
256}
257
258#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
259#[serde(rename_all = "camelCase")]
260pub struct GateViolation {
261    pub evaluator_id: String,
262    pub class: MetricClass,
263    pub decision: MetricDecision,
264    pub reason: String,
265}
266
267#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
268#[serde(rename_all = "camelCase")]
269pub struct GateDecision {
270    pub allowed: bool,
271    #[serde(default)]
272    pub violations: Vec<GateViolation>,
273}
agent_evaluate_v2_contract/report.rs

agent_evaluate_v2_contract/
report.rs
