agent_evaluate_v2_contract/
run.rs

1//! Runs and case runs: the durable side of an evaluation.
2
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5
6use crate::binding::BindingTarget;
7use crate::catalog::SnapshotRef;
8use crate::execution::{ExecutionSpec, TraceLevel};
9
10/// Agent id is an optimistic precondition; the Runtime remains the authority
11/// on the agent/runtime binding and validates it at execution time.
12#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
13#[serde(rename_all = "camelCase", deny_unknown_fields)]
14pub struct EvaluationSubject {
15    pub agent_id: String,
16    pub runtime_id: String,
17}
18
19#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase", deny_unknown_fields)]
21pub struct SnapshotSelection {
22    #[serde(rename = "ref")]
23    pub reference: SnapshotRef,
24    /// Empty runs every case of the snapshot.
25    #[serde(default)]
26    pub case_ids: Vec<String>,
27}
28
29#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
30#[serde(rename_all = "camelCase", deny_unknown_fields)]
31pub struct ExperimentVariant {
32    pub id: String,
33    pub subject_version: String,
34    #[serde(default)]
35    pub parameters: Value,
36}
37
38#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
39#[serde(rename_all = "camelCase", deny_unknown_fields)]
40pub struct CreateRunRequest {
41    pub target: BindingTarget,
42    /// Resolve the target's binding instead of listing snapshots inline.
43    #[serde(default)]
44    pub use_binding: bool,
45    #[serde(default)]
46    pub snapshots: Vec<SnapshotSelection>,
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub subject: Option<EvaluationSubject>,
49    #[serde(default)]
50    pub subject_version: String,
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub execution_override: Option<ExecutionSpec>,
53    /// Required by pairwise judges; validated at creation, not at submit.
54    #[serde(default, skip_serializing_if = "Option::is_none")]
55    pub baseline_report_id: Option<String>,
56    #[serde(default)]
57    pub seed: u64,
58    #[serde(default = "default_concurrency")]
59    pub concurrency: usize,
60    #[serde(default = "default_samples")]
61    pub samples: u32,
62    #[serde(default)]
63    pub experiment_matrix: Vec<ExperimentVariant>,
64    #[serde(default)]
65    pub idempotency_key: String,
66}
67
68fn default_concurrency() -> usize {
69    8
70}
71
72fn default_samples() -> u32 {
73    1
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
77#[serde(rename_all = "snake_case")]
78pub enum OperationState {
79    Queued,
80    Running,
81    Succeeded,
82    Failed,
83    Canceling,
84    Canceled,
85}
86
87impl OperationState {
88    pub const fn is_terminal(self) -> bool {
89        matches!(self, Self::Succeeded | Self::Failed | Self::Canceled)
90    }
91}
92
93/// Terminal classification of a case run. The five outcomes are deliberately
94/// distinct: a flaky sandbox, a broken case and a genuine quality miss demand
95/// completely different responses.
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
97#[serde(rename_all = "snake_case")]
98pub enum CaseRunState {
99    Queued,
100    Running,
101    /// Evidence stored and synchronous metrics computed.
102    Executed,
103    /// Waiting on asynchronous evaluators.
104    Analyzing,
105    Passed,
106    QualityFailed,
107    InfraFailed,
108    AnalysisFailed,
109    InvalidCase,
110    Canceled,
111}
112
113impl CaseRunState {
114    pub const fn is_terminal(self) -> bool {
115        matches!(
116            self,
117            Self::Passed
118                | Self::QualityFailed
119                | Self::InfraFailed
120                | Self::AnalysisFailed
121                | Self::InvalidCase
122                | Self::Canceled
123        )
124    }
125
126    pub const fn as_str(self) -> &'static str {
127        match self {
128            Self::Queued => "queued",
129            Self::Running => "running",
130            Self::Executed => "executed",
131            Self::Analyzing => "analyzing",
132            Self::Passed => "passed",
133            Self::QualityFailed => "quality_failed",
134            Self::InfraFailed => "infra_failed",
135            Self::AnalysisFailed => "analysis_failed",
136            Self::InvalidCase => "invalid_case",
137            Self::Canceled => "canceled",
138        }
139    }
140}
141
142/// The durable evaluate task. Its id is the `evaluateTaskId` a driver uses.
143#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
144#[serde(rename_all = "camelCase")]
145pub struct EvalOperation {
146    pub id: String,
147    pub run_id: String,
148    pub state: OperationState,
149    pub phase: String,
150    pub completed_cases: usize,
151    pub total_cases: usize,
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub report_id: Option<String>,
154    #[serde(default, skip_serializing_if = "Option::is_none")]
155    pub error_code: Option<String>,
156    pub created_at_ms: i64,
157    pub updated_at_ms: i64,
158    pub version: i64,
159}
160
161/// One planned execution of one case. `snapshot_ref` is frozen at run
162/// creation, so a later binding edit cannot retroactively change it.
163#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
164#[serde(rename_all = "camelCase")]
165pub struct CaseRunSummary {
166    pub id: String,
167    pub run_id: String,
168    #[serde(rename = "snapshotRef")]
169    pub snapshot_ref: SnapshotRef,
170    pub case_id: String,
171    pub case_digest: String,
172    pub position: usize,
173    pub state: CaseRunState,
174    pub attempt: u32,
175    #[serde(default)]
176    pub sample_index: u32,
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub variant_id: Option<String>,
179    #[serde(default, skip_serializing_if = "Option::is_none")]
180    pub sandbox_id: Option<String>,
181    #[serde(default, skip_serializing_if = "Option::is_none")]
182    pub trace_level: Option<TraceLevel>,
183    #[serde(default)]
184    pub duration_ms: u64,
185    #[serde(default, skip_serializing_if = "Option::is_none")]
186    pub error: Option<String>,
187}
188
189#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
190#[serde(rename_all = "camelCase", deny_unknown_fields)]
191pub struct ListCaseRunsRequest {
192    pub run_id: String,
193    #[serde(default, skip_serializing_if = "Option::is_none")]
194    pub state: Option<CaseRunState>,
195    #[serde(default, skip_serializing_if = "Option::is_none")]
196    pub cursor: Option<String>,
197    #[serde(default, skip_serializing_if = "Option::is_none")]
198    pub limit: Option<usize>,
199}
200
201#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
202#[serde(rename_all = "camelCase")]
203pub struct CaseRunPage {
204    pub items: Vec<CaseRunSummary>,
205    #[serde(default, skip_serializing_if = "Option::is_none")]
206    pub next_cursor: Option<String>,
207}
208
209#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
210#[serde(rename_all = "camelCase", deny_unknown_fields)]
211pub struct GetOperationRequest {
212    pub operation_id: String,
213}
214
215#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
216#[serde(rename_all = "camelCase", deny_unknown_fields)]
217pub struct CancelRunRequest {
218    pub run_id: String,
219}
agent_evaluate_v2_contract/run.rs

agent_evaluate_v2_contract/
run.rs
