agent-evaluate-v2-contract 0.1.0

//! Driver protocol shared by every executor: the in-sandbox runner, a cloud
//! runtime and the CLI's local driver all speak exactly this.

use std::collections::BTreeMap;

use serde::{Deserialize, Serialize};

use crate::EvalScope;
use crate::catalog::{EvalCase, SnapshotRef};
use crate::execution::{ExecutionSpec, TraceLevel};
use crate::run::{EvalOperation, EvaluationSubject, ExperimentVariant};

/// Sandbox handed to the driver. In-sandbox runners get their credential
/// injected at creation and see no ticket here; out-of-sandbox drivers get a
/// ticket scoped to this case run and expiring with the lease.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxAccess {
    pub sandbox_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub access_ticket_token: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub endpoint: Option<String>,
    pub expires_at_ms: i64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CaseExecutionRequest {
    pub scope: EvalScope,
    pub operation_id: String,
    pub run_id: String,
    pub case_run_id: String,
    #[serde(rename = "snapshotRef")]
    pub snapshot_ref: SnapshotRef,
    pub case: EvalCase,
    pub execution: ExecutionSpec,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub sandbox: Option<SandboxAccess>,
    /// Stable per case run, so a re-leased worker resumes the same trace.
    pub conversation_id: String,
    pub seed: u64,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant: Option<ExperimentVariant>,
    pub fencing_token: i64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimCaseRequest {
    pub evaluate_task_id: String,
    /// Fidelity this driver can actually produce. Cases demanding more are
    /// not handed out to it.
    pub trace_level: TraceLevel,
    #[serde(default = "default_lease_ms")]
    pub lease_ms: i64,
}

fn default_lease_ms() -> i64 {
    60_000
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CaseLease {
    pub execution: CaseExecutionRequest,
    pub lease_until_ms: i64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimCaseResponse {
    pub operation: EvalOperation,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub case: Option<CaseLease>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewCaseRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    #[serde(default = "default_lease_ms")]
    pub lease_ms: i64,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewCaseResponse {
    pub renewed: bool,
    pub lease_until_ms: i64,
}

/// Pointer to the trace in agent-trace. Evaluate stores the reference and the
/// derived metric values, never the spans themselves.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CaseTraceRef {
    pub conversation_id: String,
    pub run_id: String,
    #[serde(default = "default_provider")]
    pub provider: String,
    pub trace_level: TraceLevel,
    #[serde(default)]
    pub event_count: u64,
    #[serde(default)]
    pub first_event_ms: i64,
    #[serde(default)]
    pub last_event_ms: i64,
}

fn default_provider() -> String {
    "agent-trace".to_string()
}

/// Bounded evidence produced by a driver. Verdicts stay with Evaluate and are
/// a pure function of this structure plus the trace.
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CaseEvidence {
    #[serde(default)]
    pub responses: Vec<String>,
    #[serde(default)]
    pub workspace_files: BTreeMap<String, String>,
    #[serde(default)]
    pub command_exit_codes: BTreeMap<String, i32>,
    #[serde(default)]
    pub artifact_kinds: Vec<String>,
    /// Provider, protocol or authentication failure: not a quality signal.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub infrastructure_error: Option<String>,
    /// The case itself is unusable. Typed so provider text is never parsed
    /// into a terminal state.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub invalid_case_error: Option<String>,
    #[serde(default)]
    pub turns_completed: usize,
    #[serde(default)]
    pub duration_ms: u64,
    #[serde(default)]
    pub cost_micros: u64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvidenceRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    pub evidence: CaseEvidence,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub trace: Option<CaseTraceRef>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvidenceResponse {
    pub accepted: bool,
    pub operation: EvalOperation,
}