agent_evaluate_v2_contract/
catalog.rs

1//! The snapshot catalogue: the one immutable registration unit.
2
3use std::collections::BTreeMap;
4use std::fmt;
5use std::str::FromStr;
6
7use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
8
9use crate::evaluator::CaseEvaluator;
10use crate::execution::{EvalBudget, EvalTurn, ExecutionSpec, TraceLevel, VerifyCommand};
11
12/// Ownership tier. Builtin snapshots are platform assets maintained through
13/// `/admin` and readable by every tenant; tenant snapshots are private.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum OwnerKind {
17    Builtin,
18    Tenant,
19}
20
21impl OwnerKind {
22    pub const fn as_str(self) -> &'static str {
23        match self {
24            Self::Builtin => "builtin",
25            Self::Tenant => "tenant",
26        }
27    }
28}
29
30impl FromStr for OwnerKind {
31    type Err = SnapshotRefError;
32
33    fn from_str(value: &str) -> Result<Self, Self::Err> {
34        match value {
35            "builtin" => Ok(Self::Builtin),
36            "tenant" => Ok(Self::Tenant),
37            other => Err(SnapshotRefError::UnknownOwner(other.to_string())),
38        }
39    }
40}
41
42/// Builtin rows share the snapshot tables with tenant rows; the tenant and
43/// project columns carry this sentinel so the primary key keeps one shape and
44/// NULLs never take part in a unique index.
45pub const BUILTIN_SCOPE_SENTINEL: &str = "-";
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
48#[serde(rename_all = "snake_case")]
49pub enum SnapshotLifecycle {
50    /// Visible only through `/admin`; cannot be referenced or listed.
51    Draft,
52    Published,
53    /// Existing runs keep working; new runs are refused.
54    Deprecated,
55}
56
57/// Closed capability taxonomy. Cross-tenant comparison and leaderboards are
58/// aligned on this label, so it cannot be a free-form string — one capability
59/// would otherwise arrive in five spellings.
60#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
61#[serde(rename_all = "snake_case")]
62pub enum Capability {
63    ToolUse,
64    FileEdit,
65    CodeGeneration,
66    InstructionFollowing,
67    LongContext,
68    MultiTurnMemory,
69    Planning,
70    ErrorRecovery,
71    DataAnalysis,
72    WebResearch,
73    ComputerUse,
74    RefusalSafety,
75}
76
77impl Capability {
78    pub const ALL: [Self; 12] = [
79        Self::ToolUse,
80        Self::FileEdit,
81        Self::CodeGeneration,
82        Self::InstructionFollowing,
83        Self::LongContext,
84        Self::MultiTurnMemory,
85        Self::Planning,
86        Self::ErrorRecovery,
87        Self::DataAnalysis,
88        Self::WebResearch,
89        Self::ComputerUse,
90        Self::RefusalSafety,
91    ];
92
93    pub const fn as_str(self) -> &'static str {
94        match self {
95            Self::ToolUse => "tool_use",
96            Self::FileEdit => "file_edit",
97            Self::CodeGeneration => "code_generation",
98            Self::InstructionFollowing => "instruction_following",
99            Self::LongContext => "long_context",
100            Self::MultiTurnMemory => "multi_turn_memory",
101            Self::Planning => "planning",
102            Self::ErrorRecovery => "error_recovery",
103            Self::DataAnalysis => "data_analysis",
104            Self::WebResearch => "web_research",
105            Self::ComputerUse => "computer_use",
106            Self::RefusalSafety => "refusal_safety",
107        }
108    }
109}
110
111#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
112#[serde(rename_all = "snake_case")]
113pub enum Difficulty {
114    Easy,
115    Medium,
116    Hard,
117}
118
119impl Difficulty {
120    pub const fn as_str(self) -> &'static str {
121        match self {
122            Self::Easy => "easy",
123            Self::Medium => "medium",
124            Self::Hard => "hard",
125        }
126    }
127}
128
129#[derive(Debug, thiserror::Error, PartialEq, Eq)]
130pub enum SnapshotRefError {
131    #[error("snapshot reference must look like `<owner>/<id>@<version>`")]
132    Malformed,
133    #[error("unknown snapshot owner `{0}`")]
134    UnknownOwner(String),
135    #[error("snapshot reference has an empty {0}")]
136    EmptyPart(&'static str),
137}
138
139/// Immutable pointer to one snapshot version, written `builtin/tool-use@3`.
140///
141/// A mutable alias such as `latest` is deliberately unsupported: a run must
142/// record exactly what it executed.
143#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
144pub struct SnapshotRef {
145    pub owner: OwnerKind,
146    pub snapshot_id: String,
147    pub version: String,
148}
149
150impl SnapshotRef {
151    pub fn new(
152        owner: OwnerKind,
153        snapshot_id: impl Into<String>,
154        version: impl Into<String>,
155    ) -> Self {
156        Self {
157            owner,
158            snapshot_id: snapshot_id.into(),
159            version: version.into(),
160        }
161    }
162
163    pub const fn is_builtin(&self) -> bool {
164        matches!(self.owner, OwnerKind::Builtin)
165    }
166}
167
168impl fmt::Display for SnapshotRef {
169    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
170        write!(
171            formatter,
172            "{}/{}@{}",
173            self.owner.as_str(),
174            self.snapshot_id,
175            self.version
176        )
177    }
178}
179
180impl FromStr for SnapshotRef {
181    type Err = SnapshotRefError;
182
183    fn from_str(value: &str) -> Result<Self, Self::Err> {
184        let (owner, rest) = value.split_once('/').ok_or(SnapshotRefError::Malformed)?;
185        let (snapshot_id, version) = rest.split_once('@').ok_or(SnapshotRefError::Malformed)?;
186        if snapshot_id.is_empty() {
187            return Err(SnapshotRefError::EmptyPart("id"));
188        }
189        if version.is_empty() {
190            return Err(SnapshotRefError::EmptyPart("version"));
191        }
192        Ok(Self {
193            owner: owner.parse()?,
194            snapshot_id: snapshot_id.to_string(),
195            version: version.to_string(),
196        })
197    }
198}
199
200impl Serialize for SnapshotRef {
201    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
202        serializer.serialize_str(&self.to_string())
203    }
204}
205
206impl<'de> Deserialize<'de> for SnapshotRef {
207    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
208        let raw = String::deserialize(deserializer)?;
209        raw.parse().map_err(D::Error::custom)
210    }
211}
212
213/// Reference to another snapshot whose cases are copied into this version at
214/// registration time. Later releases of the referenced snapshot never change
215/// what was already frozen here.
216#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
217#[serde(rename_all = "camelCase", deny_unknown_fields)]
218pub struct SnapshotInclude {
219    #[serde(rename = "ref")]
220    pub reference: SnapshotRef,
221    /// Empty means every case of the referenced snapshot.
222    #[serde(default)]
223    pub case_ids: Vec<String>,
224    /// Applied to imported case ids; the way to resolve an id collision.
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    pub case_id_prefix: Option<String>,
227    /// Swaps the execution spec of imported cases. Their content and
228    /// evaluators stay untouched, otherwise cross-tenant comparability dies.
229    #[serde(default, skip_serializing_if = "Option::is_none")]
230    pub execution_override: Option<ExecutionSpec>,
231}
232
233#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
234#[serde(rename_all = "camelCase", deny_unknown_fields)]
235pub struct EvalCase {
236    pub case_id: String,
237    /// Server-computed content digest.
238    #[serde(default)]
239    pub digest: String,
240    pub category: String,
241    #[serde(default)]
242    pub turns: Vec<EvalTurn>,
243    /// Small immutable text fixtures materialised into the sandbox before the
244    /// case runs. Anything large stays external behind `fixture_ref`.
245    #[serde(default)]
246    pub fixtures: BTreeMap<String, String>,
247    #[serde(default, skip_serializing_if = "Option::is_none")]
248    pub fixture_ref: Option<String>,
249    #[serde(default)]
250    pub verify_commands: Vec<VerifyCommand>,
251    #[serde(default)]
252    pub budget: EvalBudget,
253    /// Overrides the snapshot default when present.
254    #[serde(default, skip_serializing_if = "Option::is_none")]
255    pub execution: Option<ExecutionSpec>,
256    /// Minimum trace fidelity a driver must offer to run this case.
257    #[serde(default = "default_trace_level")]
258    pub requires_trace_level: TraceLevel,
259    pub evaluators: Vec<CaseEvaluator>,
260    /// Set when the case arrived through an include; keeps provenance visible.
261    #[serde(default, skip_serializing_if = "Option::is_none")]
262    pub origin_ref: Option<SnapshotRef>,
263}
264
265fn default_trace_level() -> TraceLevel {
266    TraceLevel::Driver
267}
268
269#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
270#[serde(rename_all = "camelCase", deny_unknown_fields)]
271pub struct EvalSnapshot {
272    pub owner: OwnerKind,
273    /// Sentinel `-` for builtin snapshots.
274    pub tenant_id: String,
275    pub project_id: String,
276    pub snapshot_id: String,
277    pub version: String,
278    pub lifecycle: SnapshotLifecycle,
279    pub schema_version: u32,
280    pub digest: String,
281    pub capability: Capability,
282    pub difficulty: Difficulty,
283    #[serde(default)]
284    pub tags: BTreeMap<String, String>,
285    pub execution: ExecutionSpec,
286    #[serde(default)]
287    pub includes: Vec<SnapshotInclude>,
288    pub cases: Vec<EvalCase>,
289    /// Set for `--ephemeral` registrations. Expired snapshots are removed once
290    /// no in-flight run references them; sealed reports are unaffected.
291    #[serde(default, skip_serializing_if = "Option::is_none")]
292    pub ephemeral_expires_at_ms: Option<i64>,
293    pub created_at_ms: i64,
294}
295
296impl EvalSnapshot {
297    pub fn reference(&self) -> SnapshotRef {
298        SnapshotRef::new(self.owner, self.snapshot_id.clone(), self.version.clone())
299    }
300
301    pub const fn is_ephemeral(&self) -> bool {
302        self.ephemeral_expires_at_ms.is_some()
303    }
304
305    pub fn case(&self, case_id: &str) -> Option<&EvalCase> {
306        self.cases.iter().find(|case| case.case_id == case_id)
307    }
308}
309
310#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
311#[serde(rename_all = "camelCase", deny_unknown_fields)]
312pub struct RegisterSnapshotRequest {
313    pub snapshot_id: String,
314    pub version: String,
315    pub capability: Capability,
316    pub difficulty: Difficulty,
317    #[serde(default)]
318    pub tags: BTreeMap<String, String>,
319    pub execution: ExecutionSpec,
320    #[serde(default)]
321    pub includes: Vec<SnapshotInclude>,
322    #[serde(default)]
323    pub cases: Vec<EvalCase>,
324    /// Register as a temporary snapshot with a TTL. Ephemeral snapshots can
325    /// neither be bound nor included by another snapshot.
326    #[serde(default)]
327    pub ephemeral: bool,
328    #[serde(default, skip_serializing_if = "Option::is_none")]
329    pub ephemeral_ttl_ms: Option<i64>,
330    /// Optional client-computed digest. When present it must match the
331    /// server's recomputation; the server never trusts it blindly.
332    #[serde(default, skip_serializing_if = "Option::is_none")]
333    pub expected_digest: Option<String>,
334}
335
336#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
337#[serde(rename_all = "camelCase", deny_unknown_fields)]
338pub struct ListSnapshotsRequest {
339    #[serde(default, skip_serializing_if = "Option::is_none")]
340    pub owner: Option<OwnerKind>,
341    #[serde(default, skip_serializing_if = "Option::is_none")]
342    pub capability: Option<Capability>,
343    #[serde(default, skip_serializing_if = "Option::is_none")]
344    pub snapshot_id: Option<String>,
345    #[serde(default, skip_serializing_if = "Option::is_none")]
346    pub cursor: Option<String>,
347    #[serde(default, skip_serializing_if = "Option::is_none")]
348    pub limit: Option<usize>,
349}
350
351#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
352#[serde(rename_all = "camelCase")]
353pub struct SnapshotSummary {
354    #[serde(rename = "ref")]
355    pub reference: SnapshotRef,
356    pub lifecycle: SnapshotLifecycle,
357    pub capability: Capability,
358    pub difficulty: Difficulty,
359    pub digest: String,
360    pub case_count: usize,
361    #[serde(default, skip_serializing_if = "Option::is_none")]
362    pub ephemeral_expires_at_ms: Option<i64>,
363    pub created_at_ms: i64,
364}
365
366#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
367#[serde(rename_all = "camelCase")]
368pub struct SnapshotPage {
369    pub items: Vec<SnapshotSummary>,
370    #[serde(default, skip_serializing_if = "Option::is_none")]
371    pub next_cursor: Option<String>,
372}
373
374#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
375#[serde(rename_all = "camelCase", deny_unknown_fields)]
376pub struct GetSnapshotRequest {
377    #[serde(rename = "ref")]
378    pub reference: SnapshotRef,
379}
380
381#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
382#[serde(rename_all = "camelCase", deny_unknown_fields)]
383pub struct SnapshotLifecycleRequest {
384    #[serde(rename = "ref")]
385    pub reference: SnapshotRef,
386    /// Recorded on deprecation so a rejected run can name its successor.
387    #[serde(default, skip_serializing_if = "Option::is_none")]
388    pub replacement: Option<SnapshotRef>,
389}
390
391/// Portable bundle used to move builtin snapshots between Infra deployments.
392/// Import is an idempotent replay keyed on the digest.
393#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
394#[serde(rename_all = "camelCase", deny_unknown_fields)]
395pub struct SnapshotBundle {
396    pub schema_version: u32,
397    pub snapshots: Vec<EvalSnapshot>,
398    pub exported_at_ms: i64,
399}
400
401#[cfg(test)]
402mod tests {
403    use super::{OwnerKind, SnapshotRef, SnapshotRefError};
404
405    #[test]
406    fn snapshot_refs_round_trip_through_their_wire_form() {
407        let parsed: SnapshotRef = "builtin/tool-use@2026-08-20.1".parse().unwrap();
408        assert_eq!(parsed.owner, OwnerKind::Builtin);
409        assert_eq!(parsed.snapshot_id, "tool-use");
410        assert_eq!(parsed.version, "2026-08-20.1");
411        assert_eq!(parsed.to_string(), "builtin/tool-use@2026-08-20.1");
412
413        let json = serde_json::to_string(&parsed).unwrap();
414        assert_eq!(json, "\"builtin/tool-use@2026-08-20.1\"");
415        assert_eq!(serde_json::from_str::<SnapshotRef>(&json).unwrap(), parsed);
416    }
417
418    #[test]
419    fn malformed_refs_are_rejected_rather_than_silently_defaulted() {
420        assert_eq!(
421            "tool-use@1".parse::<SnapshotRef>().unwrap_err(),
422            SnapshotRefError::Malformed
423        );
424        assert_eq!(
425            "builtin/tool-use".parse::<SnapshotRef>().unwrap_err(),
426            SnapshotRefError::Malformed
427        );
428        assert_eq!(
429            "builtin/tool-use@".parse::<SnapshotRef>().unwrap_err(),
430            SnapshotRefError::EmptyPart("version")
431        );
432        assert_eq!(
433            "platform/tool-use@1".parse::<SnapshotRef>().unwrap_err(),
434            SnapshotRefError::UnknownOwner("platform".into())
435        );
436    }
437}
agent_evaluate_v2_contract/catalog.rs

agent_evaluate_v2_contract/
catalog.rs
