agent_evaluate_v2_contract/
driver.rs

1//! Driver protocol shared by every executor: the in-sandbox runner, a cloud
2//! runtime and the CLI's local driver all speak exactly this.
3
4use std::collections::BTreeMap;
5
6use serde::{Deserialize, Serialize};
7
8use crate::EvalScope;
9use crate::catalog::{EvalCase, SnapshotRef};
10use crate::execution::{ExecutionSpec, TraceLevel};
11use crate::run::{EvalOperation, EvaluationSubject, ExperimentVariant};
12
13/// Sandbox handed to the driver. In-sandbox runners get their credential
14/// injected at creation and see no ticket here; out-of-sandbox drivers get a
15/// ticket scoped to this case run and expiring with the lease.
16#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
17#[serde(rename_all = "camelCase", deny_unknown_fields)]
18pub struct SandboxAccess {
19    pub sandbox_id: String,
20    #[serde(default, skip_serializing_if = "Option::is_none")]
21    pub access_ticket_token: Option<String>,
22    #[serde(default, skip_serializing_if = "Option::is_none")]
23    pub endpoint: Option<String>,
24    pub expires_at_ms: i64,
25}
26
27#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
28#[serde(rename_all = "camelCase", deny_unknown_fields)]
29pub struct CaseExecutionRequest {
30    pub scope: EvalScope,
31    pub operation_id: String,
32    pub run_id: String,
33    pub case_run_id: String,
34    #[serde(rename = "snapshotRef")]
35    pub snapshot_ref: SnapshotRef,
36    pub case: EvalCase,
37    pub execution: ExecutionSpec,
38    #[serde(default, skip_serializing_if = "Option::is_none")]
39    pub subject: Option<EvaluationSubject>,
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub sandbox: Option<SandboxAccess>,
42    /// Stable per case run, so a re-leased worker resumes the same trace.
43    pub conversation_id: String,
44    pub seed: u64,
45    #[serde(default)]
46    pub sample_index: u32,
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub variant: Option<ExperimentVariant>,
49    pub fencing_token: i64,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
53#[serde(rename_all = "camelCase", deny_unknown_fields)]
54pub struct ClaimCaseRequest {
55    pub evaluate_task_id: String,
56    /// Fidelity this driver can actually produce. Cases demanding more are
57    /// not handed out to it.
58    pub trace_level: TraceLevel,
59    #[serde(default = "default_lease_ms")]
60    pub lease_ms: i64,
61}
62
63fn default_lease_ms() -> i64 {
64    60_000
65}
66
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase", deny_unknown_fields)]
69pub struct CaseLease {
70    pub execution: CaseExecutionRequest,
71    pub lease_until_ms: i64,
72}
73
74#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
75#[serde(rename_all = "camelCase", deny_unknown_fields)]
76pub struct ClaimCaseResponse {
77    pub operation: EvalOperation,
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub case: Option<CaseLease>,
80}
81
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
83#[serde(rename_all = "camelCase", deny_unknown_fields)]
84pub struct RenewCaseRequest {
85    pub evaluate_task_id: String,
86    pub case_run_id: String,
87    pub fencing_token: i64,
88    #[serde(default = "default_lease_ms")]
89    pub lease_ms: i64,
90}
91
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
93#[serde(rename_all = "camelCase", deny_unknown_fields)]
94pub struct RenewCaseResponse {
95    pub renewed: bool,
96    pub lease_until_ms: i64,
97}
98
99/// Pointer to the trace in agent-trace. Evaluate stores the reference and the
100/// derived metric values, never the spans themselves.
101#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
102#[serde(rename_all = "camelCase", deny_unknown_fields)]
103pub struct CaseTraceRef {
104    pub conversation_id: String,
105    pub run_id: String,
106    #[serde(default = "default_provider")]
107    pub provider: String,
108    pub trace_level: TraceLevel,
109    #[serde(default)]
110    pub event_count: u64,
111    #[serde(default)]
112    pub first_event_ms: i64,
113    #[serde(default)]
114    pub last_event_ms: i64,
115}
116
117fn default_provider() -> String {
118    "agent-trace".to_string()
119}
120
121/// Bounded evidence produced by a driver. Verdicts stay with Evaluate and are
122/// a pure function of this structure plus the trace.
123#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
124#[serde(rename_all = "camelCase", deny_unknown_fields)]
125pub struct CaseEvidence {
126    #[serde(default)]
127    pub responses: Vec<String>,
128    #[serde(default)]
129    pub workspace_files: BTreeMap<String, String>,
130    #[serde(default)]
131    pub command_exit_codes: BTreeMap<String, i32>,
132    #[serde(default)]
133    pub artifact_kinds: Vec<String>,
134    /// Provider, protocol or authentication failure: not a quality signal.
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub infrastructure_error: Option<String>,
137    /// The case itself is unusable. Typed so provider text is never parsed
138    /// into a terminal state.
139    #[serde(default, skip_serializing_if = "Option::is_none")]
140    pub invalid_case_error: Option<String>,
141    #[serde(default)]
142    pub turns_completed: usize,
143    #[serde(default)]
144    pub duration_ms: u64,
145    #[serde(default)]
146    pub cost_micros: u64,
147}
148
149#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
150#[serde(rename_all = "camelCase", deny_unknown_fields)]
151pub struct SubmitEvidenceRequest {
152    pub evaluate_task_id: String,
153    pub case_run_id: String,
154    pub fencing_token: i64,
155    pub evidence: CaseEvidence,
156    #[serde(default, skip_serializing_if = "Option::is_none")]
157    pub trace: Option<CaseTraceRef>,
158}
159
160#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
161#[serde(rename_all = "camelCase", deny_unknown_fields)]
162pub struct SubmitEvidenceResponse {
163    pub accepted: bool,
164    pub operation: EvalOperation,
165}
agent_evaluate_v2_contract/driver.rs

agent_evaluate_v2_contract/
driver.rs
