agent_evaluate_v2_contract/
evaluator.rs

1//! Evaluators: the domain rules attached to a case.
2//!
3//! An evaluator is inlined on its case because it is only meaningful for that
4//! case. Cross-snapshot comparability comes from `(evaluatorId, digest)`, not
5//! from a separate catalog resource nobody remembers to version.
6
7use serde::{Deserialize, Serialize};
8use serde_json::Value;
9
10/// How a metric participates in release decisions.
11#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum MetricClass {
14    /// Quality we actively push up. Regressions gate on statistical evidence.
15    Goal,
16    /// Must never break. A single failure gates without a significance test.
17    Guardrail,
18    /// Observed only. Never contributes to a case verdict or a gate.
19    #[default]
20    Diagnostic,
21}
22
23impl MetricClass {
24    pub const fn affects_case_verdict(self) -> bool {
25        matches!(self, Self::Goal | Self::Guardrail)
26    }
27
28    pub const fn as_str(self) -> &'static str {
29        match self {
30            Self::Goal => "goal",
31            Self::Guardrail => "guardrail",
32            Self::Diagnostic => "diagnostic",
33        }
34    }
35}
36
37#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum MetricDecision {
40    BlockRelease,
41    RollBack,
42    Investigate,
43    #[default]
44    ObserveOnly,
45}
46
47impl MetricDecision {
48    pub const fn is_blocking(self) -> bool {
49        matches!(self, Self::BlockRelease | Self::RollBack)
50    }
51}
52
53#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
54#[serde(rename_all = "camelCase", deny_unknown_fields)]
55pub struct MetricSpec {
56    #[serde(default)]
57    pub class: MetricClass,
58    #[serde(default)]
59    pub decision: MetricDecision,
60    /// Contacted when the metric is recommended for retirement.
61    #[serde(default)]
62    pub owner: String,
63    /// Guardrails leave this unset: staying green forever is their job.
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub retire_after_idle_runs: Option<u32>,
66}
67
68/// Deterministic assertion over bounded evidence.
69#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
70#[serde(
71    tag = "kind",
72    rename_all = "snake_case",
73    rename_all_fields = "camelCase",
74    deny_unknown_fields
75)]
76pub enum TypedAssertion {
77    OutputContains {
78        turn: usize,
79        values: Vec<String>,
80        #[serde(default)]
81        any: bool,
82    },
83    WorkspaceFile {
84        path: String,
85        #[serde(default)]
86        contains: Vec<String>,
87        #[serde(default, skip_serializing_if = "Option::is_none")]
88        max_bytes: Option<usize>,
89    },
90    CommandExit {
91        name: String,
92        expected_exit_code: i32,
93    },
94    ArtifactExists {
95        artifact_kind: String,
96    },
97}
98
99/// Deterministic metric computed from the case's trace (plus evidence).
100///
101/// Bounds are inclusive. A metric with neither bound set is observational: it
102/// records a value without ever failing, which is a legitimate way to collect
103/// a signal before deciding what "good" means.
104#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
105#[serde(
106    tag = "kind",
107    rename_all = "snake_case",
108    rename_all_fields = "camelCase",
109    deny_unknown_fields
110)]
111pub enum TraceMetricSpec {
112    EventCount {
113        event_type: String,
114        #[serde(default, skip_serializing_if = "Option::is_none")]
115        min: Option<u64>,
116        #[serde(default, skip_serializing_if = "Option::is_none")]
117        max: Option<u64>,
118    },
119    ToolCallCount {
120        #[serde(default, skip_serializing_if = "Option::is_none")]
121        tool: Option<String>,
122        #[serde(default, skip_serializing_if = "Option::is_none")]
123        min: Option<u64>,
124        #[serde(default, skip_serializing_if = "Option::is_none")]
125        max: Option<u64>,
126    },
127    ToolErrorRate {
128        #[serde(default, skip_serializing_if = "Option::is_none")]
129        max: Option<f64>,
130    },
131    TurnCount {
132        #[serde(default, skip_serializing_if = "Option::is_none")]
133        min: Option<u64>,
134        #[serde(default, skip_serializing_if = "Option::is_none")]
135        max: Option<u64>,
136    },
137    LatencyMs {
138        #[serde(default, skip_serializing_if = "Option::is_none")]
139        max: Option<u64>,
140    },
141    TokenCostMicros {
142        #[serde(default, skip_serializing_if = "Option::is_none")]
143        max: Option<u64>,
144    },
145    RetryCount {
146        #[serde(default, skip_serializing_if = "Option::is_none")]
147        max: Option<u64>,
148    },
149    /// Same `(tool, argsDigest)` repeated more than `max_repeats` in a row.
150    LoopDetected { max_repeats: u64 },
151    /// Listed event types must appear in this relative order.
152    EventSequence { event_types: Vec<String> },
153}
154
155impl TraceMetricSpec {
156    /// Stable id used as the metric's storage key within one evaluator.
157    pub fn metric_id(&self) -> String {
158        match self {
159            Self::EventCount { event_type, .. } => format!("event_count:{event_type}"),
160            Self::ToolCallCount { tool, .. } => match tool {
161                Some(tool) => format!("tool_call_count:{tool}"),
162                None => "tool_call_count".to_string(),
163            },
164            Self::ToolErrorRate { .. } => "tool_error_rate".to_string(),
165            Self::TurnCount { .. } => "turn_count".to_string(),
166            Self::LatencyMs { .. } => "latency_ms".to_string(),
167            Self::TokenCostMicros { .. } => "token_cost_micros".to_string(),
168            Self::RetryCount { .. } => "retry_count".to_string(),
169            Self::LoopDetected { .. } => "loop_detected".to_string(),
170            Self::EventSequence { .. } => "event_sequence".to_string(),
171        }
172    }
173
174    /// True when the metric needs authoritative tool/retry/token data, which
175    /// only a runtime-written trace can provide.
176    pub const fn requires_runtime_trace(&self) -> bool {
177        matches!(
178            self,
179            Self::ToolCallCount { .. }
180                | Self::ToolErrorRate { .. }
181                | Self::TokenCostMicros { .. }
182                | Self::RetryCount { .. }
183                | Self::LoopDetected { .. }
184                | Self::EventSequence { .. }
185                | Self::EventCount { .. }
186        )
187    }
188}
189
190#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
191#[serde(rename_all = "snake_case")]
192pub enum JudgeKind {
193    /// Absolute score against a rubric.
194    Rubric,
195    /// Compared against a baseline output.
196    Pairwise,
197    /// Compared against a reference answer.
198    Reference,
199}
200
201#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
202#[serde(rename_all = "snake_case")]
203pub enum JudgeAggregation {
204    /// Median of the per-judge scores. Robust to one outlier judge.
205    Median,
206    WeightedMean,
207    /// Majority vote; requires an odd number of at least three judges.
208    Majority,
209    /// Everyone must agree; requires at least two judges.
210    Unanimous,
211}
212
213#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
214#[serde(rename_all = "camelCase", deny_unknown_fields)]
215pub struct JudgeModel {
216    pub model_ref: String,
217    #[serde(default = "default_weight")]
218    pub weight: f64,
219}
220
221fn default_weight() -> f64 {
222    1.0
223}
224
225#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
226#[serde(rename_all = "camelCase", deny_unknown_fields)]
227pub struct RubricCriterion {
228    pub id: String,
229    pub weight: f64,
230}
231
232#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
233#[serde(rename_all = "camelCase", deny_unknown_fields)]
234pub struct RubricScale {
235    pub min: f64,
236    pub max: f64,
237}
238
239#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
240#[serde(rename_all = "camelCase", deny_unknown_fields)]
241pub struct Rubric {
242    pub criteria: Vec<RubricCriterion>,
243    pub scale: RubricScale,
244}
245
246/// What a judge is allowed to look at. Anything requiring a trace summary
247/// forces the case to demand a runtime-level trace.
248#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
249#[serde(rename_all = "camelCase", deny_unknown_fields)]
250pub struct JudgeInputs {
251    #[serde(default)]
252    pub output: bool,
253    #[serde(default)]
254    pub trace_summary: bool,
255    #[serde(default)]
256    pub workspace_files: Vec<String>,
257}
258
259/// Model-graded evaluator. Everything that can move a verdict is pinned and
260/// digested, so judge drift can never be mistaken for subject regression.
261#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
262#[serde(rename_all = "camelCase", deny_unknown_fields)]
263pub struct LlmJudgeSpec {
264    pub judge: JudgeKind,
265    /// One entry is a single judge; more than one is a cross-model ensemble.
266    pub judges: Vec<JudgeModel>,
267    pub aggregation: JudgeAggregation,
268    /// Inter-judge agreement below this marks the result `low_agreement`,
269    /// which drops it out of gating and into the human review queue.
270    #[serde(default = "default_min_agreement")]
271    pub min_agreement: f64,
272    pub prompt_digest: String,
273    #[serde(default, skip_serializing_if = "Option::is_none")]
274    pub rubric: Option<Rubric>,
275    #[serde(default, skip_serializing_if = "Option::is_none")]
276    pub pass_threshold: Option<f64>,
277    /// Self-consistency samples per judge, aggregated within the judge first.
278    #[serde(default = "default_samples")]
279    pub samples: u32,
280    #[serde(default)]
281    pub temperature: f64,
282    /// Pairwise only: run both orderings to cancel position bias.
283    #[serde(default)]
284    pub position_swap: bool,
285    /// Reference only: pointer to the expected answer.
286    #[serde(default, skip_serializing_if = "Option::is_none")]
287    pub reference_ref: Option<String>,
288    /// Budget for the whole ensemble, not per judge.
289    #[serde(default)]
290    pub max_cost_micros: u64,
291    #[serde(default)]
292    pub inputs: JudgeInputs,
293}
294
295fn default_min_agreement() -> f64 {
296    0.66
297}
298
299fn default_samples() -> u32 {
300    1
301}
302
303#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
304#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
305pub enum EvaluatorImplementation {
306    /// Deterministic assertions over bounded evidence.
307    Assertions { assertions: Vec<TypedAssertion> },
308    /// Deterministic metrics derived from the case's trace.
309    TraceMetrics { metrics: Vec<TraceMetricSpec> },
310    /// Model-graded, executed by the isolated executor.
311    LlmJudge(LlmJudgeSpec),
312    /// Custom evaluator executed by the isolated executor; the control plane
313    /// only maps an attested result and never loads plugin code.
314    Plugin {
315        plugin_ref: String,
316        #[serde(default)]
317        config: Value,
318    },
319}
320
321impl EvaluatorImplementation {
322    pub const fn is_asynchronous(&self) -> bool {
323        matches!(self, Self::LlmJudge(_) | Self::Plugin { .. })
324    }
325
326    pub const fn kind_str(&self) -> &'static str {
327        match self {
328            Self::Assertions { .. } => "assertions",
329            Self::TraceMetrics { .. } => "trace_metrics",
330            Self::LlmJudge(_) => "llm_judge",
331            Self::Plugin { .. } => "plugin",
332        }
333    }
334}
335
336#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
337#[serde(rename_all = "camelCase", deny_unknown_fields)]
338pub struct CaseEvaluator {
339    pub evaluator_id: String,
340    /// Server-computed. A client-supplied value is recomputed and compared.
341    #[serde(default)]
342    pub digest: String,
343    #[serde(default)]
344    pub metric: MetricSpec,
345    pub implementation: EvaluatorImplementation,
346}
347
348/// Minimum human-label evidence before a model-graded evaluator may gate.
349/// Applied to the aggregated ensemble output, not to an individual judge.
350pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
351pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;
352
353/// Platform-maintained judge catalogue entry. Only models on this list may
354/// back a gating evaluator; a tenant may still judge with any visible model
355/// as long as the metric stays diagnostic.
356#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
357#[serde(rename_all = "camelCase", deny_unknown_fields)]
358pub struct JudgeAllowlistEntry {
359    pub model_ref: String,
360    /// False while the model is still accumulating calibration evidence.
361    #[serde(default)]
362    pub gate_approved: bool,
363    #[serde(default)]
364    pub labeled_samples: u64,
365    #[serde(default)]
366    pub human_agreement: f64,
367    #[serde(default)]
368    pub cost_tier: String,
369    #[serde(default)]
370    pub notes: String,
371    pub updated_at_ms: i64,
372}
373
374#[cfg(test)]
375mod tests {
376    use super::{MetricClass, TraceMetricSpec, TypedAssertion};
377    use serde_json::json;
378
379    #[test]
380    fn assertions_use_snake_case_tags_and_camel_case_fields() {
381        let value = serde_json::to_value(TypedAssertion::OutputContains {
382            turn: 0,
383            values: vec!["ok".into()],
384            any: false,
385        })
386        .unwrap();
387        assert_eq!(
388            value,
389            json!({ "kind": "output_contains", "turn": 0, "values": ["ok"], "any": false })
390        );
391    }
392
393    #[test]
394    fn trace_metric_ids_are_stable_and_distinguish_per_tool_slices() {
395        let all = TraceMetricSpec::ToolCallCount {
396            tool: None,
397            min: None,
398            max: None,
399        };
400        let one = TraceMetricSpec::ToolCallCount {
401            tool: Some("bash".into()),
402            min: None,
403            max: None,
404        };
405        assert_eq!(all.metric_id(), "tool_call_count");
406        assert_eq!(one.metric_id(), "tool_call_count:bash");
407    }
408
409    #[test]
410    fn diagnostic_is_the_default_class_so_new_signals_cannot_gate_by_accident() {
411        assert_eq!(MetricClass::default(), MetricClass::Diagnostic);
412        assert!(!MetricClass::Diagnostic.affects_case_verdict());
413    }
414}
agent_evaluate_v2_contract/evaluator.rs

agent_evaluate_v2_contract/
evaluator.rs
