agent-evaluate-v2-contract 0.1.0

//! Runs and case runs: the durable side of an evaluation.

use serde::{Deserialize, Serialize};
use serde_json::Value;

use crate::binding::BindingTarget;
use crate::catalog::SnapshotRef;
use crate::execution::{ExecutionSpec, TraceLevel};

/// Agent id is an optimistic precondition; the Runtime remains the authority
/// on the agent/runtime binding and validates it at execution time.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluationSubject {
    pub agent_id: String,
    pub runtime_id: String,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SnapshotSelection {
    #[serde(rename = "ref")]
    pub reference: SnapshotRef,
    /// Empty runs every case of the snapshot.
    #[serde(default)]
    pub case_ids: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ExperimentVariant {
    pub id: String,
    pub subject_version: String,
    #[serde(default)]
    pub parameters: Value,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CreateRunRequest {
    pub target: BindingTarget,
    /// Resolve the target's binding instead of listing snapshots inline.
    #[serde(default)]
    pub use_binding: bool,
    #[serde(default)]
    pub snapshots: Vec<SnapshotSelection>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default)]
    pub subject_version: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub execution_override: Option<ExecutionSpec>,
    /// Required by pairwise judges; validated at creation, not at submit.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub baseline_report_id: Option<String>,
    #[serde(default)]
    pub seed: u64,
    #[serde(default = "default_concurrency")]
    pub concurrency: usize,
    #[serde(default = "default_samples")]
    pub samples: u32,
    #[serde(default)]
    pub experiment_matrix: Vec<ExperimentVariant>,
    #[serde(default)]
    pub idempotency_key: String,
}

fn default_concurrency() -> usize {
    8
}

fn default_samples() -> u32 {
    1
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OperationState {
    Queued,
    Running,
    Succeeded,
    Failed,
    Canceling,
    Canceled,
}

impl OperationState {
    pub const fn is_terminal(self) -> bool {
        matches!(self, Self::Succeeded | Self::Failed | Self::Canceled)
    }
}

/// Terminal classification of a case run. The five outcomes are deliberately
/// distinct: a flaky sandbox, a broken case and a genuine quality miss demand
/// completely different responses.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseRunState {
    Queued,
    Running,
    /// Evidence stored and synchronous metrics computed.
    Executed,
    /// Waiting on asynchronous evaluators.
    Analyzing,
    Passed,
    QualityFailed,
    InfraFailed,
    AnalysisFailed,
    InvalidCase,
    Canceled,
}

impl CaseRunState {
    pub const fn is_terminal(self) -> bool {
        matches!(
            self,
            Self::Passed
                | Self::QualityFailed
                | Self::InfraFailed
                | Self::AnalysisFailed
                | Self::InvalidCase
                | Self::Canceled
        )
    }

    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Queued => "queued",
            Self::Running => "running",
            Self::Executed => "executed",
            Self::Analyzing => "analyzing",
            Self::Passed => "passed",
            Self::QualityFailed => "quality_failed",
            Self::InfraFailed => "infra_failed",
            Self::AnalysisFailed => "analysis_failed",
            Self::InvalidCase => "invalid_case",
            Self::Canceled => "canceled",
        }
    }
}

/// The durable evaluate task. Its id is the `evaluateTaskId` a driver uses.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalOperation {
    pub id: String,
    pub run_id: String,
    pub state: OperationState,
    pub phase: String,
    pub completed_cases: usize,
    pub total_cases: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub report_id: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error_code: Option<String>,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
    pub version: i64,
}

/// One planned execution of one case. `snapshot_ref` is frozen at run
/// creation, so a later binding edit cannot retroactively change it.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunSummary {
    pub id: String,
    pub run_id: String,
    #[serde(rename = "snapshotRef")]
    pub snapshot_ref: SnapshotRef,
    pub case_id: String,
    pub case_digest: String,
    pub position: usize,
    pub state: CaseRunState,
    pub attempt: u32,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub sandbox_id: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub trace_level: Option<TraceLevel>,
    #[serde(default)]
    pub duration_ms: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListCaseRunsRequest {
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<CaseRunState>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub limit: Option<usize>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunPage {
    pub items: Vec<CaseRunSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetOperationRequest {
    pub operation_id: String,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CancelRunRequest {
    pub run_id: String,
}