agent-evaluate-contract 0.3.0

//! HTTP API contract for the Evaluate service.
//!
//! This service provides a programmatic interface for running agent evaluation
//! cases, collecting metrics, and producing reports. The eval harness replays
//! predefined conversation turns against an agent runtime instance and judges
//! outcomes against oracles.
//!
//! # Layering
//!
//! ```text
//! agent-evaluate-contract   paths + DTOs        (this crate)
//! agent-evaluate-runtime    eval runner + judge
//! agent-evaluate-server     router + auth + bin
//! ```

use serde::{Deserialize, Serialize};
use serde_json::Value;

pub const SERVICE_NAME: &str = "agent-evaluate";
pub const DEFAULT_PORT: u16 = 5150;
pub const AUDIENCE: &str = "agent-evaluate";
pub const SERVICE_TOKEN_ENV: &str = "AGENT_EVALUATE_SERVICE_TOKEN";
pub const MAX_RESOURCE_ID_CHARS: usize = 191;

pub const HEALTH_PATH: &str = "/health";
pub const RUN_EVAL_PATH: &str = "/internal/v1/evaluate/run";
pub const LIST_CASES_PATH: &str = "/internal/v1/evaluate/cases";
pub const GET_CASE_PATH: &str = "/internal/v1/evaluate/cases/get";
pub const GET_REPORT_PATH: &str = "/internal/v1/evaluate/reports";
pub const LOAD_CASES_PATH: &str = "/internal/v1/evaluate/cases/load";
pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
pub const ADMIN_OVERVIEW_PATH: &str = "/internal/v1/evaluate/admin/overview";
pub const ADMIN_SUITES_PATH: &str = "/internal/v1/evaluate/admin/suites";
pub const ADMIN_RUNS_PATH: &str = "/internal/v1/evaluate/admin/runs";
pub const ADMIN_RUN_CASES_PATH: &str = "/internal/v1/evaluate/admin/runs/cases";
pub const ADMIN_CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/admin/runs/cancel";
pub const ADMIN_REPORT_PATH: &str = "/internal/v1/evaluate/admin/reports/get";
pub const ADMIN_REVIEWS_PATH: &str = "/internal/v1/evaluate/admin/reviews";
pub const ADMIN_CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/claim";
pub const ADMIN_COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/complete";

// ---------------------------------------------------------------------------
// Eval case types
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalCaseFile {
    pub schema_version: i32,
    pub cases: Vec<EvalCase>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalCase {
    pub id: String,
    pub category: String,
    #[serde(default)]
    pub notes: String,
    pub turns: Vec<EvalTurn>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub workspace_files: Option<Value>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub oracle: Option<EvalOracle>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub verify_commands: Option<Vec<VerifyCommand>>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub trace_assertions: Option<Value>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub outcome_assertions: Option<Value>,
    #[serde(default)]
    pub expects_artifact: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub generated_workspace_files: Option<Value>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub workspace_copies: Option<Value>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub workspace_patches: Option<Value>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalTurn {
    pub user: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalOracle {
    pub kind: String,
    #[serde(default)]
    pub expected_values: Vec<Value>,
    #[serde(default)]
    pub notes: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VerifyCommand {
    pub name: String,
    pub command: String,
    #[serde(default)]
    pub expected_exit_code: i32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub output_contains: Option<Vec<String>>,
}

// ---------------------------------------------------------------------------
// API request/response types
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RunEvalRequest {
    #[serde(default)]
    pub case_ids: Vec<String>,
    #[serde(default)]
    pub categories: Vec<String>,
    #[serde(default)]
    pub runtime_types: Vec<String>,
    #[serde(default)]
    pub max_turns: usize,
    #[serde(default)]
    pub turn_timeout_ms: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalReport {
    pub id: String,
    pub total_cases: usize,
    pub passed_cases: usize,
    pub failed_cases: usize,
    pub infrastructure_failure_cases: usize,
    /// Cases whose immutable schema/fixture/evaluator configuration could not
    /// produce a meaningful quality verdict. Kept separate from provider or
    /// runtime infrastructure failures.
    #[serde(default)]
    pub invalid_case_count: usize,
    pub non_infrastructure_pass_rate: f64,
    pub case_results: Vec<EvalCaseResult>,
    pub created_at: String,
    pub duration_ms: u64,
    #[serde(default)]
    pub total_cost_micros: u64,
    #[serde(default)]
    pub p95_case_duration_ms: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalCaseResult {
    pub case_id: String,
    pub category: String,
    pub passed: bool,
    pub infrastructure_failure: bool,
    /// Structured terminal classification for non-quality failures. The
    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub failure_kind: Option<CaseFailureKind>,
    pub turns_completed: usize,
    pub total_turns: usize,
    #[serde(default)]
    pub oracle_results: Vec<OracleVerdict>,
    #[serde(default)]
    pub verify_results: Vec<VerifyCommandResult>,
    #[serde(default)]
    pub trace_assertion_results: Vec<AssertionVerdict>,
    #[serde(default)]
    pub outcome_assertion_results: Vec<AssertionVerdict>,
    /// Versioned evaluator outputs. Legacy inline assertions continue to use
    /// outcome_assertion_results during migration.
    #[serde(default)]
    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    pub duration_ms: u64,
    #[serde(default)]
    pub cost_micros: u64,
    /// Durable experiment identity. Legacy synchronous runs leave these at
    /// their defaults; durable reports retain them so repeated samples from
    /// different variants are never conflated as flakiness.
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseFailureKind {
    InfraFailed,
    InvalidCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct OracleVerdict {
    pub expected: String,
    pub actual: String,
    pub matched: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VerifyCommandResult {
    pub name: String,
    pub exit_code: i32,
    pub expected_exit_code: i32,
    pub passed: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub output: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssertionVerdict {
    pub rule: String,
    pub passed: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub detail: Option<String>,
}

/// Strictly parsed compatibility schema for `outcomeAssertions`.
///
/// `EvalCase` retains the original JSON field so v1 wire JSON is unchanged;
/// evaluators must deserialize it into this type and fail closed on malformed
/// or unknown rules.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct OutcomeAssertionSpec {
    #[serde(default)]
    pub turns: Vec<TurnAssertionSpec>,
    #[serde(default)]
    pub workspace_files: Vec<WorkspaceFileAssertionSpec>,
    #[serde(default)]
    pub required_changed_paths: Vec<String>,
    #[serde(default)]
    pub forbid_workspace_changes: bool,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct TurnAssertionSpec {
    pub index: usize,
    #[serde(default)]
    pub contains: Vec<String>,
    #[serde(default)]
    pub contains_any: Vec<Vec<String>>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct WorkspaceFileAssertionSpec {
    pub path: String,
    #[serde(default)]
    pub contains: Vec<String>,
    #[serde(default)]
    pub contains_any: Vec<Vec<String>>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_chars: Option<usize>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max_lines: Option<usize>,
}

/// Versioned typed assertion schema used by durable suites. The legacy
/// `Value` fields on [`EvalCase`] remain only for v1 wire compatibility.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(
    tag = "kind",
    rename_all = "snake_case",
    rename_all_fields = "camelCase",
    deny_unknown_fields
)]
pub enum TypedAssertion {
    OutputContains {
        turn: usize,
        values: Vec<String>,
        #[serde(default)]
        any: bool,
    },
    WorkspaceFile {
        path: String,
        #[serde(default)]
        contains: Vec<String>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_bytes: Option<usize>,
    },
    CommandExit {
        name: String,
        command_ref: String,
        expected_exit_code: i32,
    },
    TraceEvent {
        event_type: String,
        #[serde(default)]
        minimum_count: usize,
    },
    ArtifactExists {
        artifact_kind: String,
    },
    /// A custom evaluator that is executed by the isolated executor. The
    /// control plane accepts only an attested result for the exact immutable
    /// plugin reference; it never loads plugin code into the service process.
    Plugin {
        plugin_ref: String,
        #[serde(default)]
        config: Value,
    },
}

/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct VersionedResourceRef {
    pub id: String,
    pub version: String,
}

/// Agent is an optimistic precondition; Runtime ID remains the execution
/// selector and Runtime is the authoritative owner of their binding.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluationSubject {
    pub agent_id: String,
    pub runtime_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterEnvironmentProfileRequest {
    pub profile_id: String,
    pub version: String,
    /// Existing Workspace Environment Space selected for the Task. Provisioned
    /// environments can be added later behind the same immutable profile.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub space_id: Option<String>,
    #[serde(default)]
    pub metadata: std::collections::BTreeMap<String, String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EnvironmentProfileVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub profile_id: String,
    pub version: String,
    pub digest: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub space_id: Option<String>,
    #[serde(default)]
    pub metadata: std::collections::BTreeMap<String, String>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvaluatorImplementation {
    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
    BuiltinAssertions { assertions: Vec<TypedAssertion> },
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterEvaluatorVersionRequest {
    pub evaluator_id: String,
    pub version: String,
    pub implementation: EvaluatorImplementation,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluatorVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub evaluator_id: String,
    pub version: String,
    pub digest: String,
    pub implementation: EvaluatorImplementation,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvaluatorVerdict {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub passed: bool,
    #[serde(default)]
    pub assertions: Vec<AssertionVerdict>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct VersionedEvalCase {
    pub id: String,
    pub category: String,
    #[serde(default)]
    pub turns: Vec<EvalTurn>,
    /// Small, immutable text fixtures materialised into the selected Space
    /// before Runtime executes the case. Large fixtures remain external and
    /// are referenced through `fixtureRef`.
    #[serde(default)]
    pub workspace_files: std::collections::BTreeMap<String, String>,
    #[serde(default)]
    pub assertions: Vec<TypedAssertion>,
    /// Versioned environment selection. Omitted only for legacy suites.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<VersionedResourceRef>,
    /// Domain-owned evaluator policies executed after evidence collection.
    #[serde(default)]
    pub evaluator_versions: Vec<VersionedResourceRef>,
    #[serde(default)]
    pub verify_commands: Vec<VerifyCommand>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub fixture_ref: Option<String>,
    #[serde(default)]
    pub budget: EvalBudget,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalBudget {
    pub timeout_ms: u64,
    pub max_output_bytes: usize,
    pub max_cost_micros: u64,
}

impl Default for EvalBudget {
    fn default() -> Self {
        Self {
            timeout_ms: 300_000,
            max_output_bytes: 8 * 1024 * 1024,
            max_cost_micros: 0,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalSuiteVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub cases: Vec<VersionedEvalCase>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterSuiteRequest {
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub cases: Vec<VersionedEvalCase>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OperationState {
    Queued,
    Running,
    Succeeded,
    Failed,
    Canceling,
    Canceled,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseRunState {
    Queued,
    Running,
    Passed,
    QualityFailed,
    InfraFailed,
    InvalidCase,
    Canceled,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CreateEvalRunRequest {
    pub suite_id: String,
    pub suite_version: String,
    pub subject_version: String,
    pub executor_profile: String,
    /// Concrete execution target. New Runtime Task based runs require it;
    /// subject_version remains as a compatibility/report label.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default)]
    pub case_ids: Vec<String>,
    #[serde(default)]
    pub seed: u64,
    #[serde(default = "default_concurrency")]
    pub concurrency: usize,
    #[serde(default)]
    pub idempotency_key: String,
    /// Independent deterministic samples per `(case, variant)`.
    #[serde(default = "default_samples")]
    pub samples: u32,
    /// Empty preserves the v1 single-subject run. A non-empty matrix expands
    /// the run without requiring consumers to coordinate many operations.
    #[serde(default)]
    pub experiment_matrix: Vec<ExperimentVariant>,
}

fn default_concurrency() -> usize {
    8
}

fn default_samples() -> u32 {
    1
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ExperimentVariant {
    pub id: String,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default)]
    pub parameters: Value,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalOperation {
    pub id: String,
    pub run_id: String,
    pub state: OperationState,
    pub phase: String,
    pub attempt: u32,
    pub completed_cases: usize,
    pub total_cases: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub report_id: Option<String>,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
    pub version: i64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_poll_after_ms: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error_code: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetOperationRequest {
    pub operation_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CancelRunRequest {
    pub run_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListCaseRunsRequest {
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<CaseRunState>,
    #[serde(default = "default_case_page")]
    pub limit: usize,
}

fn default_case_page() -> usize {
    50
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunSummary {
    pub id: String,
    pub run_id: String,
    pub case_id: String,
    pub position: usize,
    pub state: CaseRunState,
    pub attempt: u32,
    pub duration_ms: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    #[serde(default)]
    pub evidence_refs: Vec<EvidenceRef>,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunPage {
    pub items: Vec<CaseRunSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalAccessScope {
    pub tenant_id: String,
    pub project_id: String,
    pub subject: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxExecutionRequest {
    /// Tenant/project ownership is passed explicitly so the isolated executor
    /// can only finalize evidence in the namespace later validated by the
    /// control plane.
    pub scope: EvalAccessScope,
    pub operation_id: String,
    pub run_id: String,
    pub case_run_id: String,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<EnvironmentProfileVersion>,
    #[serde(default)]
    pub evaluator_versions: Vec<EvaluatorVersion>,
    pub seed: u64,
    pub case: VersionedEvalCase,
    pub fencing_token: i64,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant: Option<ExperimentVariant>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct PluginEvaluationEvidence {
    pub plugin_ref: String,
    pub passed: bool,
    #[serde(default)]
    pub detail: String,
    /// Attestation produced by the configured sandbox boundary. Production
    /// evaluation fails closed when it is absent.
    pub attestation: String,
}

/// Immutable evidence object owned by the scoped Evaluate namespace.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvidenceRef {
    pub reference: String,
    pub tenant_id: String,
    pub project_id: String,
    pub run_id: String,
    pub case_run_id: String,
    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
    pub digest: String,
    pub size_bytes: u64,
    /// Attestation bound to the isolated executor response boundary.
    pub attestation: String,
}

/// Bounded evidence returned by the isolated executor. Verdict calculation
/// remains owned by Evaluate and is deterministic from this typed structure.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxExecutionEvidence {
    #[serde(default)]
    pub responses: Vec<String>,
    #[serde(default)]
    pub workspace_files: std::collections::BTreeMap<String, String>,
    #[serde(default)]
    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
    #[serde(default)]
    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
    #[serde(default)]
    pub artifact_kinds: Vec<String>,
    /// Immutable blob/object references owned by Evaluate's evidence namespace.
    #[serde(default)]
    pub evidence_refs: Vec<EvidenceRef>,
    #[serde(default)]
    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub infrastructure_error: Option<String>,
    /// A sandbox may explicitly reject an immutable case as unsupported or
    /// invalid. This is deliberately typed so provider text is never parsed as
    /// a terminal-state code.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub invalid_case_error: Option<String>,
    #[serde(default)]
    pub duration_ms: u64,
    /// Executor-measured cost in millionths of the configured billing unit.
    #[serde(default)]
    pub cost_micros: u64,
}

/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
/// never needs a callback URL for the Runtime.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimEvaluationCaseRequest {
    pub evaluate_task_id: String,
    #[serde(default = "default_driver_lease_ms")]
    pub lease_ms: i64,
}

fn default_driver_lease_ms() -> i64 {
    60_000
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluationCaseLease {
    pub execution: SandboxExecutionRequest,
    pub lease_until_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimEvaluationCaseResponse {
    pub operation: EvalOperation,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub case: Option<EvaluationCaseLease>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewEvaluationCaseRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    #[serde(default = "default_driver_lease_ms")]
    pub lease_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewEvaluationCaseResponse {
    pub renewed: bool,
    pub lease_until_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvaluationEvidenceRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    pub evidence: SandboxExecutionEvidence,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvaluationEvidenceResponse {
    pub accepted: bool,
    pub operation: EvalOperation,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ListCasesRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub category: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ListCasesResponse {
    pub cases: Vec<EvalCaseSummary>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalCaseSummary {
    pub id: String,
    pub category: String,
    pub notes: String,
    pub turns: usize,
    #[serde(default)]
    pub has_oracle: bool,
    #[serde(default)]
    pub has_verify_commands: bool,
    #[serde(default)]
    pub expects_artifact: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GetCaseRequest {
    pub case_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct LoadCasesRequest {
    pub file_path: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GetReportRequest {
    pub report_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CompareReportsRequest {
    pub report_id: String,
    pub baseline_report_id: String,
    #[serde(default)]
    pub minimum_pass_rate: f64,
    #[serde(default = "default_max_infra_rate")]
    pub maximum_infra_failure_rate: f64,
    #[serde(default = "default_significance")]
    pub significance_level: f64,
    #[serde(default)]
    pub minimum_effect: f64,
}

fn default_max_infra_rate() -> f64 {
    1.0
}

fn default_significance() -> f64 {
    0.05
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseOutcomeDiff {
    pub case_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
    pub baseline_passed: bool,
    pub current_passed: bool,
    pub flaky_signal: bool,
    #[serde(default)]
    pub historical_flake_rate: f64,
    #[serde(default)]
    pub historical_samples: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ReportComparison {
    pub report_id: String,
    pub baseline_report_id: String,
    pub pass_rate_delta: f64,
    pub infra_failure_rate_delta: f64,
    pub pass_rate_confidence_low: f64,
    pub pass_rate_confidence_high: f64,
    pub effect_confidence_low: f64,
    pub effect_confidence_high: f64,
    pub p_value: f64,
    pub statistically_significant: bool,
    pub changed_cases: Vec<CaseOutcomeDiff>,
    pub gate_passed: bool,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReviewState {
    Queued,
    Claimed,
    Approved,
    Rejected,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct HumanReview {
    pub id: String,
    pub run_id: String,
    pub case_run_id: String,
    pub state: ReviewState,
    pub reason: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub reviewer: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub decision_note: Option<String>,
    pub fencing_token: i64,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EnqueueReviewRequest {
    pub run_id: String,
    pub case_run_id: String,
    pub reason: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimReviewRequest {
    #[serde(default = "default_review_lease")]
    pub lease_ms: i64,
}

fn default_review_lease() -> i64 {
    300_000
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CompleteReviewRequest {
    pub review_id: String,
    pub fencing_token: i64,
    pub approved: bool,
    #[serde(default)]
    pub note: String,
}

// ---------------------------------------------------------------------------
// Evaluate administration query model
// ---------------------------------------------------------------------------

pub const DEFAULT_ADMIN_PAGE_SIZE: usize = 50;
pub const MAX_ADMIN_PAGE_SIZE: usize = 200;

fn default_admin_page() -> usize {
    DEFAULT_ADMIN_PAGE_SIZE
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminSuitesRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminSuitesRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminRunsRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<OperationState>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminRunsRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            state: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminReviewsRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<ReviewState>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminReviewsRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            state: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvaluateAdminOverview {
    pub suite_versions: u64,
    pub run_states: std::collections::BTreeMap<String, u64>,
    pub case_states: std::collections::BTreeMap<String, u64>,
    pub review_states: std::collections::BTreeMap<String, u64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminSuiteSummary {
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub case_count: usize,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminSuitePage {
    pub items: Vec<AdminSuiteSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminRunSummary {
    pub run_id: String,
    pub operation_id: String,
    pub suite_id: String,
    pub suite_version: String,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    pub state: OperationState,
    pub phase: String,
    pub completed_cases: usize,
    pub total_cases: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub report_id: Option<String>,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
    pub version: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminRunPage {
    pub items: Vec<AdminRunSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminReviewPage {
    pub items: Vec<HumanReview>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

/// Error body shared by every non-2xx response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiError {
    pub status: u16,
    pub message: String,
}

#[cfg(test)]
mod tests {
    use super::{
        DEFAULT_ADMIN_PAGE_SIZE, ListAdminReviewsRequest, ListAdminRunsRequest,
        ListAdminSuitesRequest, TypedAssertion,
    };
    use serde_json::json;

    #[test]
    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
        let assertion = TypedAssertion::TraceEvent {
            event_type: "task.completed".into(),
            minimum_count: 1,
        };
        let value = serde_json::to_value(&assertion).unwrap();
        assert_eq!(
            value,
            json!({
                "kind": "trace_event",
                "eventType": "task.completed",
                "minimumCount": 1
            })
        );
        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
    }

    #[test]
    fn admin_queries_use_the_same_bounded_default_in_rust_and_serde() {
        assert_eq!(
            ListAdminSuitesRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        assert_eq!(
            ListAdminRunsRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        assert_eq!(
            ListAdminReviewsRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        let decoded: ListAdminRunsRequest = serde_json::from_value(json!({})).unwrap();
        assert_eq!(decoded.limit, DEFAULT_ADMIN_PAGE_SIZE);
    }
}