agent_evaluate_v2_contract/
execution.rs

1//! How a case is executed: which sandbox it gets and where the agent runs.
2
3use std::collections::BTreeMap;
4
5use serde::{Deserialize, Serialize};
6
7/// Where the agent process lives. The evidence, trace and metric models are
8/// identical for both; only the agent's location and credential delivery
9/// differ. `mode` participates in the case digest so results produced under
10/// the two semantics are never merged into one trend line.
11#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum ExecutionMode {
14    /// Agent runs inside the sandbox and operates local files/processes.
15    InSandbox,
16    /// Agent runs outside and drives the same sandbox through Workspace APIs.
17    OutSandbox,
18}
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
21#[serde(rename_all = "snake_case")]
22pub enum SpaceAccessMode {
23    ReadOnly,
24    ReadWrite,
25}
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28#[serde(rename_all = "snake_case")]
29pub enum SandboxPlacement {
30    Computer(String),
31    ProviderPool(String),
32}
33
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "camelCase", deny_unknown_fields)]
36pub struct SandboxMountSpec {
37    pub alias: String,
38    pub space_id: String,
39    pub mode: SpaceAccessMode,
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub path_prefix: Option<String>,
42}
43
44/// Declarative subset of the Workspace `CreateSandboxRequest`. Evaluate never
45/// builds images or templates; it only references what Workspace published.
46#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
47#[serde(rename_all = "camelCase", deny_unknown_fields)]
48pub struct WorkspaceSpec {
49    pub provider: String,
50    pub placement: SandboxPlacement,
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub template_id: Option<String>,
53    #[serde(default, skip_serializing_if = "Option::is_none")]
54    pub template_version: Option<String>,
55    /// Workspace image snapshot. Deliberately not called `snapshotId`: that
56    /// name belongs to the Eval Snapshot and conflating them is a bug factory.
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub image_snapshot_id: Option<String>,
59    #[serde(default)]
60    pub mounts: Vec<SandboxMountSpec>,
61    pub ttl_ms: u64,
62    #[serde(default)]
63    pub profile: BTreeMap<String, String>,
64}
65
66/// In-sandbox agent entrypoint. Credentials are injected at sandbox creation
67/// and never baked into the image, so this carries no secrets.
68#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
69#[serde(rename_all = "camelCase", deny_unknown_fields)]
70pub struct AgentSpec {
71    pub entrypoint: String,
72    #[serde(default)]
73    pub env: BTreeMap<String, String>,
74}
75
76#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
77#[serde(rename_all = "camelCase", deny_unknown_fields)]
78pub struct ExecutionSpec {
79    pub mode: ExecutionMode,
80    pub workspace: WorkspaceSpec,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub agent: Option<AgentSpec>,
83}
84
85/// Fidelity of the trace a driver can produce. A metric may never claim more
86/// than its trace can support, so this is part of the contract rather than an
87/// operational detail.
88#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
89#[serde(rename_all = "snake_case")]
90pub enum TraceLevel {
91    /// An external driver reconstructs turn boundaries; tool calls are only
92    /// as trustworthy as the agent's self-report.
93    Driver,
94    /// The runtime writes trace events itself: tool calls, retries, per-turn
95    /// latency and token usage are all authoritative.
96    Runtime,
97}
98
99impl TraceLevel {
100    /// `Runtime` satisfies a `Driver` requirement, never the other way round.
101    pub const fn satisfies(self, required: Self) -> bool {
102        (self as u8) >= (required as u8)
103    }
104
105    pub const fn as_str(self) -> &'static str {
106        match self {
107            Self::Driver => "driver",
108            Self::Runtime => "runtime",
109        }
110    }
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
114#[serde(rename_all = "camelCase", deny_unknown_fields)]
115pub struct EvalTurn {
116    pub user: String,
117}
118
119#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
120#[serde(rename_all = "camelCase", deny_unknown_fields)]
121pub struct VerifyCommand {
122    pub name: String,
123    pub command: String,
124    #[serde(default)]
125    pub expected_exit_code: i32,
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
129#[serde(rename_all = "camelCase", deny_unknown_fields)]
130pub struct EvalBudget {
131    pub timeout_ms: u64,
132    pub max_output_bytes: usize,
133    pub max_cost_micros: u64,
134}
135
136impl Default for EvalBudget {
137    fn default() -> Self {
138        Self {
139            timeout_ms: 300_000,
140            max_output_bytes: 8 * 1024 * 1024,
141            max_cost_micros: 0,
142        }
143    }
144}
145
146#[cfg(test)]
147mod tests {
148    use super::TraceLevel;
149
150    #[test]
151    fn runtime_trace_satisfies_driver_requirement_but_not_the_reverse() {
152        assert!(TraceLevel::Runtime.satisfies(TraceLevel::Driver));
153        assert!(TraceLevel::Runtime.satisfies(TraceLevel::Runtime));
154        assert!(TraceLevel::Driver.satisfies(TraceLevel::Driver));
155        assert!(!TraceLevel::Driver.satisfies(TraceLevel::Runtime));
156    }
157}
agent_evaluate_v2_contract/execution.rs

agent_evaluate_v2_contract/
execution.rs
