agent-evaluate-v2-contract 0.1.0

//! The snapshot catalogue: the one immutable registration unit.

use std::collections::BTreeMap;
use std::fmt;
use std::str::FromStr;

use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};

use crate::evaluator::CaseEvaluator;
use crate::execution::{EvalBudget, EvalTurn, ExecutionSpec, TraceLevel, VerifyCommand};

/// Ownership tier. Builtin snapshots are platform assets maintained through
/// `/admin` and readable by every tenant; tenant snapshots are private.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OwnerKind {
    Builtin,
    Tenant,
}

impl OwnerKind {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Builtin => "builtin",
            Self::Tenant => "tenant",
        }
    }
}

impl FromStr for OwnerKind {
    type Err = SnapshotRefError;

    fn from_str(value: &str) -> Result<Self, Self::Err> {
        match value {
            "builtin" => Ok(Self::Builtin),
            "tenant" => Ok(Self::Tenant),
            other => Err(SnapshotRefError::UnknownOwner(other.to_string())),
        }
    }
}

/// Builtin rows share the snapshot tables with tenant rows; the tenant and
/// project columns carry this sentinel so the primary key keeps one shape and
/// NULLs never take part in a unique index.
pub const BUILTIN_SCOPE_SENTINEL: &str = "-";

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SnapshotLifecycle {
    /// Visible only through `/admin`; cannot be referenced or listed.
    Draft,
    Published,
    /// Existing runs keep working; new runs are refused.
    Deprecated,
}

/// Closed capability taxonomy. Cross-tenant comparison and leaderboards are
/// aligned on this label, so it cannot be a free-form string — one capability
/// would otherwise arrive in five spellings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Capability {
    ToolUse,
    FileEdit,
    CodeGeneration,
    InstructionFollowing,
    LongContext,
    MultiTurnMemory,
    Planning,
    ErrorRecovery,
    DataAnalysis,
    WebResearch,
    ComputerUse,
    RefusalSafety,
}

impl Capability {
    pub const ALL: [Self; 12] = [
        Self::ToolUse,
        Self::FileEdit,
        Self::CodeGeneration,
        Self::InstructionFollowing,
        Self::LongContext,
        Self::MultiTurnMemory,
        Self::Planning,
        Self::ErrorRecovery,
        Self::DataAnalysis,
        Self::WebResearch,
        Self::ComputerUse,
        Self::RefusalSafety,
    ];

    pub const fn as_str(self) -> &'static str {
        match self {
            Self::ToolUse => "tool_use",
            Self::FileEdit => "file_edit",
            Self::CodeGeneration => "code_generation",
            Self::InstructionFollowing => "instruction_following",
            Self::LongContext => "long_context",
            Self::MultiTurnMemory => "multi_turn_memory",
            Self::Planning => "planning",
            Self::ErrorRecovery => "error_recovery",
            Self::DataAnalysis => "data_analysis",
            Self::WebResearch => "web_research",
            Self::ComputerUse => "computer_use",
            Self::RefusalSafety => "refusal_safety",
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Difficulty {
    Easy,
    Medium,
    Hard,
}

impl Difficulty {
    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Easy => "easy",
            Self::Medium => "medium",
            Self::Hard => "hard",
        }
    }
}

#[derive(Debug, thiserror::Error, PartialEq, Eq)]
pub enum SnapshotRefError {
    #[error("snapshot reference must look like `<owner>/<id>@<version>`")]
    Malformed,
    #[error("unknown snapshot owner `{0}`")]
    UnknownOwner(String),
    #[error("snapshot reference has an empty {0}")]
    EmptyPart(&'static str),
}

/// Immutable pointer to one snapshot version, written `builtin/tool-use@3`.
///
/// A mutable alias such as `latest` is deliberately unsupported: a run must
/// record exactly what it executed.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SnapshotRef {
    pub owner: OwnerKind,
    pub snapshot_id: String,
    pub version: String,
}

impl SnapshotRef {
    pub fn new(
        owner: OwnerKind,
        snapshot_id: impl Into<String>,
        version: impl Into<String>,
    ) -> Self {
        Self {
            owner,
            snapshot_id: snapshot_id.into(),
            version: version.into(),
        }
    }

    pub const fn is_builtin(&self) -> bool {
        matches!(self.owner, OwnerKind::Builtin)
    }
}

impl fmt::Display for SnapshotRef {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            formatter,
            "{}/{}@{}",
            self.owner.as_str(),
            self.snapshot_id,
            self.version
        )
    }
}

impl FromStr for SnapshotRef {
    type Err = SnapshotRefError;

    fn from_str(value: &str) -> Result<Self, Self::Err> {
        let (owner, rest) = value.split_once('/').ok_or(SnapshotRefError::Malformed)?;
        let (snapshot_id, version) = rest.split_once('@').ok_or(SnapshotRefError::Malformed)?;
        if snapshot_id.is_empty() {
            return Err(SnapshotRefError::EmptyPart("id"));
        }
        if version.is_empty() {
            return Err(SnapshotRefError::EmptyPart("version"));
        }
        Ok(Self {
            owner: owner.parse()?,
            snapshot_id: snapshot_id.to_string(),
            version: version.to_string(),
        })
    }
}

impl Serialize for SnapshotRef {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        serializer.serialize_str(&self.to_string())
    }
}

impl<'de> Deserialize<'de> for SnapshotRef {
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        let raw = String::deserialize(deserializer)?;
        raw.parse().map_err(D::Error::custom)
    }
}

/// Reference to another snapshot whose cases are copied into this version at
/// registration time. Later releases of the referenced snapshot never change
/// what was already frozen here.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SnapshotInclude {
    #[serde(rename = "ref")]
    pub reference: SnapshotRef,
    /// Empty means every case of the referenced snapshot.
    #[serde(default)]
    pub case_ids: Vec<String>,
    /// Applied to imported case ids; the way to resolve an id collision.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub case_id_prefix: Option<String>,
    /// Swaps the execution spec of imported cases. Their content and
    /// evaluators stay untouched, otherwise cross-tenant comparability dies.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub execution_override: Option<ExecutionSpec>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalCase {
    pub case_id: String,
    /// Server-computed content digest.
    #[serde(default)]
    pub digest: String,
    pub category: String,
    #[serde(default)]
    pub turns: Vec<EvalTurn>,
    /// Small immutable text fixtures materialised into the sandbox before the
    /// case runs. Anything large stays external behind `fixture_ref`.
    #[serde(default)]
    pub fixtures: BTreeMap<String, String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub fixture_ref: Option<String>,
    #[serde(default)]
    pub verify_commands: Vec<VerifyCommand>,
    #[serde(default)]
    pub budget: EvalBudget,
    /// Overrides the snapshot default when present.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub execution: Option<ExecutionSpec>,
    /// Minimum trace fidelity a driver must offer to run this case.
    #[serde(default = "default_trace_level")]
    pub requires_trace_level: TraceLevel,
    pub evaluators: Vec<CaseEvaluator>,
    /// Set when the case arrived through an include; keeps provenance visible.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub origin_ref: Option<SnapshotRef>,
}

fn default_trace_level() -> TraceLevel {
    TraceLevel::Driver
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalSnapshot {
    pub owner: OwnerKind,
    /// Sentinel `-` for builtin snapshots.
    pub tenant_id: String,
    pub project_id: String,
    pub snapshot_id: String,
    pub version: String,
    pub lifecycle: SnapshotLifecycle,
    pub schema_version: u32,
    pub digest: String,
    pub capability: Capability,
    pub difficulty: Difficulty,
    #[serde(default)]
    pub tags: BTreeMap<String, String>,
    pub execution: ExecutionSpec,
    #[serde(default)]
    pub includes: Vec<SnapshotInclude>,
    pub cases: Vec<EvalCase>,
    /// Set for `--ephemeral` registrations. Expired snapshots are removed once
    /// no in-flight run references them; sealed reports are unaffected.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ephemeral_expires_at_ms: Option<i64>,
    pub created_at_ms: i64,
}

impl EvalSnapshot {
    pub fn reference(&self) -> SnapshotRef {
        SnapshotRef::new(self.owner, self.snapshot_id.clone(), self.version.clone())
    }

    pub const fn is_ephemeral(&self) -> bool {
        self.ephemeral_expires_at_ms.is_some()
    }

    pub fn case(&self, case_id: &str) -> Option<&EvalCase> {
        self.cases.iter().find(|case| case.case_id == case_id)
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterSnapshotRequest {
    pub snapshot_id: String,
    pub version: String,
    pub capability: Capability,
    pub difficulty: Difficulty,
    #[serde(default)]
    pub tags: BTreeMap<String, String>,
    pub execution: ExecutionSpec,
    #[serde(default)]
    pub includes: Vec<SnapshotInclude>,
    #[serde(default)]
    pub cases: Vec<EvalCase>,
    /// Register as a temporary snapshot with a TTL. Ephemeral snapshots can
    /// neither be bound nor included by another snapshot.
    #[serde(default)]
    pub ephemeral: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ephemeral_ttl_ms: Option<i64>,
    /// Optional client-computed digest. When present it must match the
    /// server's recomputation; the server never trusts it blindly.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_digest: Option<String>,
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListSnapshotsRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub owner: Option<OwnerKind>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub capability: Option<Capability>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub snapshot_id: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub limit: Option<usize>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SnapshotSummary {
    #[serde(rename = "ref")]
    pub reference: SnapshotRef,
    pub lifecycle: SnapshotLifecycle,
    pub capability: Capability,
    pub difficulty: Difficulty,
    pub digest: String,
    pub case_count: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ephemeral_expires_at_ms: Option<i64>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SnapshotPage {
    pub items: Vec<SnapshotSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetSnapshotRequest {
    #[serde(rename = "ref")]
    pub reference: SnapshotRef,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SnapshotLifecycleRequest {
    #[serde(rename = "ref")]
    pub reference: SnapshotRef,
    /// Recorded on deprecation so a rejected run can name its successor.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub replacement: Option<SnapshotRef>,
}

/// Portable bundle used to move builtin snapshots between Infra deployments.
/// Import is an idempotent replay keyed on the digest.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SnapshotBundle {
    pub schema_version: u32,
    pub snapshots: Vec<EvalSnapshot>,
    pub exported_at_ms: i64,
}

#[cfg(test)]
mod tests {
    use super::{OwnerKind, SnapshotRef, SnapshotRefError};

    #[test]
    fn snapshot_refs_round_trip_through_their_wire_form() {
        let parsed: SnapshotRef = "builtin/tool-use@2026-08-20.1".parse().unwrap();
        assert_eq!(parsed.owner, OwnerKind::Builtin);
        assert_eq!(parsed.snapshot_id, "tool-use");
        assert_eq!(parsed.version, "2026-08-20.1");
        assert_eq!(parsed.to_string(), "builtin/tool-use@2026-08-20.1");

        let json = serde_json::to_string(&parsed).unwrap();
        assert_eq!(json, "\"builtin/tool-use@2026-08-20.1\"");
        assert_eq!(serde_json::from_str::<SnapshotRef>(&json).unwrap(), parsed);
    }

    #[test]
    fn malformed_refs_are_rejected_rather_than_silently_defaulted() {
        assert_eq!(
            "tool-use@1".parse::<SnapshotRef>().unwrap_err(),
            SnapshotRefError::Malformed
        );
        assert_eq!(
            "builtin/tool-use".parse::<SnapshotRef>().unwrap_err(),
            SnapshotRefError::Malformed
        );
        assert_eq!(
            "builtin/tool-use@".parse::<SnapshotRef>().unwrap_err(),
            SnapshotRefError::EmptyPart("version")
        );
        assert_eq!(
            "platform/tool-use@1".parse::<SnapshotRef>().unwrap_err(),
            SnapshotRefError::UnknownOwner("platform".into())
        );
    }
}