agent-evaluate-v2-contract 0.1.0

//! How a case is executed: which sandbox it gets and where the agent runs.

use std::collections::BTreeMap;

use serde::{Deserialize, Serialize};

/// Where the agent process lives. The evidence, trace and metric models are
/// identical for both; only the agent's location and credential delivery
/// differ. `mode` participates in the case digest so results produced under
/// the two semantics are never merged into one trend line.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ExecutionMode {
    /// Agent runs inside the sandbox and operates local files/processes.
    InSandbox,
    /// Agent runs outside and drives the same sandbox through Workspace APIs.
    OutSandbox,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SpaceAccessMode {
    ReadOnly,
    ReadWrite,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SandboxPlacement {
    Computer(String),
    ProviderPool(String),
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxMountSpec {
    pub alias: String,
    pub space_id: String,
    pub mode: SpaceAccessMode,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub path_prefix: Option<String>,
}

/// Declarative subset of the Workspace `CreateSandboxRequest`. Evaluate never
/// builds images or templates; it only references what Workspace published.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct WorkspaceSpec {
    pub provider: String,
    pub placement: SandboxPlacement,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub template_id: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub template_version: Option<String>,
    /// Workspace image snapshot. Deliberately not called `snapshotId`: that
    /// name belongs to the Eval Snapshot and conflating them is a bug factory.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub image_snapshot_id: Option<String>,
    #[serde(default)]
    pub mounts: Vec<SandboxMountSpec>,
    pub ttl_ms: u64,
    #[serde(default)]
    pub profile: BTreeMap<String, String>,
}

/// In-sandbox agent entrypoint. Credentials are injected at sandbox creation
/// and never baked into the image, so this carries no secrets.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AgentSpec {
    pub entrypoint: String,
    #[serde(default)]
    pub env: BTreeMap<String, String>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ExecutionSpec {
    pub mode: ExecutionMode,
    pub workspace: WorkspaceSpec,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub agent: Option<AgentSpec>,
}

/// Fidelity of the trace a driver can produce. A metric may never claim more
/// than its trace can support, so this is part of the contract rather than an
/// operational detail.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TraceLevel {
    /// An external driver reconstructs turn boundaries; tool calls are only
    /// as trustworthy as the agent's self-report.
    Driver,
    /// The runtime writes trace events itself: tool calls, retries, per-turn
    /// latency and token usage are all authoritative.
    Runtime,
}

impl TraceLevel {
    /// `Runtime` satisfies a `Driver` requirement, never the other way round.
    pub const fn satisfies(self, required: Self) -> bool {
        (self as u8) >= (required as u8)
    }

    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Driver => "driver",
            Self::Runtime => "runtime",
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalTurn {
    pub user: String,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct VerifyCommand {
    pub name: String,
    pub command: String,
    #[serde(default)]
    pub expected_exit_code: i32,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalBudget {
    pub timeout_ms: u64,
    pub max_output_bytes: usize,
    pub max_cost_micros: u64,
}

impl Default for EvalBudget {
    fn default() -> Self {
        Self {
            timeout_ms: 300_000,
            max_output_bytes: 8 * 1024 * 1024,
            max_cost_micros: 0,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::TraceLevel;

    #[test]
    fn runtime_trace_satisfies_driver_requirement_but_not_the_reverse() {
        assert!(TraceLevel::Runtime.satisfies(TraceLevel::Driver));
        assert!(TraceLevel::Runtime.satisfies(TraceLevel::Runtime));
        assert!(TraceLevel::Driver.satisfies(TraceLevel::Driver));
        assert!(!TraceLevel::Driver.satisfies(TraceLevel::Runtime));
    }
}