agent_evaluate_v2_contract/
analysis.rs

1//! Asynchronous analysis: model-graded judges, plugins and re-analysis.
2//!
3//! Re-analysis is the payoff of storing a trace reference instead of a
4//! verdict: a new analyzer version can rescore an existing case run without
5//! executing the agent again.
6
7use serde::{Deserialize, Serialize};
8
9use crate::EvalScope;
10use crate::catalog::SnapshotRef;
11use crate::evaluator::CaseEvaluator;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "snake_case")]
15pub enum AnalysisState {
16    Queued,
17    Running,
18    Succeeded,
19    Failed,
20}
21
22#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
23#[serde(rename_all = "camelCase", deny_unknown_fields)]
24pub struct AnalysisTask {
25    pub id: String,
26    pub scope: EvalScope,
27    pub run_id: String,
28    pub case_run_id: String,
29    #[serde(rename = "snapshotRef")]
30    pub snapshot_ref: SnapshotRef,
31    pub case_id: String,
32    pub evaluator: CaseEvaluator,
33    /// Report revision this result will land in.
34    pub revision: u32,
35    pub state: AnalysisState,
36    pub attempt: u32,
37    pub fencing_token: i64,
38    pub created_at_ms: i64,
39}
40
41#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase", deny_unknown_fields)]
43pub struct ClaimAnalysisRequest {
44    #[serde(default = "default_lease_ms")]
45    pub lease_ms: i64,
46    #[serde(default = "default_batch")]
47    pub max_tasks: usize,
48}
49
50fn default_lease_ms() -> i64 {
51    120_000
52}
53
54fn default_batch() -> usize {
55    1
56}
57
58#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
59#[serde(rename_all = "camelCase", deny_unknown_fields)]
60pub struct ClaimAnalysisResponse {
61    pub tasks: Vec<AnalysisTask>,
62    pub lease_until_ms: i64,
63}
64
65/// One judge's contribution. Kept per judge so a later question — "which
66/// judge was dragging the score?" — is answerable from stored data.
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase", deny_unknown_fields)]
69pub struct JudgeVote {
70    pub model_ref: String,
71    /// Median of this judge's own self-consistency samples.
72    pub score: f64,
73    #[serde(default)]
74    pub cost_micros: u64,
75    #[serde(default)]
76    pub detail: String,
77}
78
79#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
80#[serde(rename_all = "camelCase", deny_unknown_fields)]
81pub struct JudgeEvidence {
82    pub votes: Vec<JudgeVote>,
83    /// Attestation bound to the isolated executor boundary. Production fails
84    /// closed without it; the control plane never runs judge code itself.
85    #[serde(default)]
86    pub attestation: String,
87    /// Set when the executor stopped early on the ensemble cost budget.
88    #[serde(default)]
89    pub budget_exhausted: bool,
90}
91
92#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
93#[serde(rename_all = "camelCase", deny_unknown_fields)]
94pub struct PluginEvidence {
95    pub plugin_ref: String,
96    pub passed: bool,
97    #[serde(default)]
98    pub detail: String,
99    #[serde(default)]
100    pub cost_micros: u64,
101    #[serde(default)]
102    pub attestation: String,
103}
104
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
106#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
107pub enum AnalysisOutcome {
108    Judge(JudgeEvidence),
109    Plugin(PluginEvidence),
110    /// The analyzer could not produce a result. Retryable, and deliberately
111    /// separate from a quality failure.
112    Failed {
113        error: String,
114    },
115}
116
117#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
118#[serde(rename_all = "camelCase", deny_unknown_fields)]
119pub struct SubmitAnalysisRequest {
120    pub task_id: String,
121    pub fencing_token: i64,
122    pub outcome: AnalysisOutcome,
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
126#[serde(rename_all = "camelCase", deny_unknown_fields)]
127pub struct SubmitAnalysisResponse {
128    pub accepted: bool,
129}
130
131/// Rescore stored case runs with a new analyzer set. Produces a new report
132/// revision; the executed evidence and trace reference are untouched.
133#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
134#[serde(rename_all = "camelCase", deny_unknown_fields)]
135pub struct RerunAnalysisRequest {
136    pub run_id: String,
137    /// Empty rescoring means "every asynchronous evaluator of every case".
138    #[serde(default)]
139    pub case_run_ids: Vec<String>,
140    #[serde(default)]
141    pub evaluators: Vec<CaseEvaluator>,
142}
143
144#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
145#[serde(rename_all = "camelCase")]
146pub struct RerunAnalysisResponse {
147    pub revision: u32,
148    pub scheduled_tasks: usize,
149}
agent_evaluate_v2_contract/analysis.rs

agent_evaluate_v2_contract/
analysis.rs
