agent-evaluate-contract 0.4.0

//! HTTP API contract for the Evaluate service.
//!
//! Evaluate is a control plane over four immutable catalog resources —
//! Case, Environment Profile, Evaluator and Suite — plus the durable Task
//! that a Runtime pulls from. Case is the node that owns the dependency
//! edges, so it carries its own id, version and digest like every other
//! catalog resource; a Suite is only an ordered selection of Case versions.
//!
//! # Layering
//!
//! ```text
//! agent-evaluate-contract   paths + DTOs        (this crate)
//! agent-evaluate-runtime    control plane + verdicts
//! agent-evaluate-server     router + auth + bin
//! ```

pub mod gate;
pub mod metric;
pub mod review;

pub use gate::*;
pub use metric::*;
pub use review::*;

use serde::{Deserialize, Serialize};
use serde_json::Value;

pub const SERVICE_NAME: &str = "agent-evaluate";
pub const DEFAULT_PORT: u16 = 5150;
pub const AUDIENCE: &str = "agent-evaluate";
pub const MAX_RESOURCE_ID_CHARS: usize = 191;

pub const HEALTH_PATH: &str = "/health";
pub const CASE_VERSIONS_PATH: &str = "/internal/v1/evaluate/case-versions";
pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
pub const SAMPLE_REVIEWS_PATH: &str = "/internal/v1/evaluate/reviews/sample";
pub const METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/metrics/health";
pub const METRIC_CANDIDATES_PATH: &str = "/internal/v1/evaluate/metrics/candidates";
pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
pub const ADMIN_OVERVIEW_PATH: &str = "/internal/v1/evaluate/admin/overview";
pub const ADMIN_CASES_PATH: &str = "/internal/v1/evaluate/admin/cases";
pub const ADMIN_SUITES_PATH: &str = "/internal/v1/evaluate/admin/suites";
pub const ADMIN_RUNS_PATH: &str = "/internal/v1/evaluate/admin/runs";
pub const ADMIN_RUN_CASES_PATH: &str = "/internal/v1/evaluate/admin/runs/cases";
pub const ADMIN_CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/admin/runs/cancel";
pub const ADMIN_REPORT_PATH: &str = "/internal/v1/evaluate/admin/reports/get";
pub const ADMIN_REVIEWS_PATH: &str = "/internal/v1/evaluate/admin/reviews";
pub const ADMIN_CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/claim";
pub const ADMIN_COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/complete";
pub const ADMIN_METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/admin/metrics/health";

// ---------------------------------------------------------------------------
// Eval case types
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalTurn {
    pub user: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct VerifyCommand {
    pub name: String,
    pub command: String,
    #[serde(default)]
    pub expected_exit_code: i32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub output_contains: Option<Vec<String>>,
}

// ---------------------------------------------------------------------------
// API request/response types
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalReport {
    pub id: String,
    pub total_cases: usize,
    pub passed_cases: usize,
    pub failed_cases: usize,
    pub infrastructure_failure_cases: usize,
    /// Cases whose immutable schema/fixture/evaluator configuration could not
    /// produce a meaningful quality verdict. Kept separate from provider or
    /// runtime infrastructure failures.
    #[serde(default)]
    pub invalid_case_count: usize,
    pub non_infrastructure_pass_rate: f64,
    /// Per-evaluator slices. The aggregate pass rate above cannot say which
    /// dimension moved; these can.
    #[serde(default)]
    pub metrics: Vec<MetricSummary>,
    pub case_results: Vec<EvalCaseResult>,
    pub created_at: String,
    pub duration_ms: u64,
    #[serde(default)]
    pub total_cost_micros: u64,
    /// Judge cost, reported apart from the subject's execution cost.
    #[serde(default)]
    pub judge_cost_micros: u64,
    #[serde(default)]
    pub p95_case_duration_ms: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalCaseResult {
    pub case_id: String,
    /// Immutable Case version this result was produced from. Trend, flake and
    /// regression statistics key on the pair, never on the bare id.
    pub case_version: String,
    pub category: String,
    pub passed: bool,
    pub infrastructure_failure: bool,
    /// Structured terminal classification for non-quality failures. The
    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub failure_kind: Option<CaseFailureKind>,
    pub turns_completed: usize,
    pub total_turns: usize,
    /// Verdicts of the Case's own inline assertions.
    #[serde(default)]
    pub assertion_results: Vec<AssertionVerdict>,
    /// Versioned evaluator outputs.
    #[serde(default)]
    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    pub duration_ms: u64,
    #[serde(default)]
    pub cost_micros: u64,
    /// Durable experiment identity, retained so repeated samples from
    /// different variants are never conflated as flakiness.
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseFailureKind {
    InfraFailed,
    InvalidCase,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssertionVerdict {
    pub rule: String,
    pub passed: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub detail: Option<String>,
}

/// Versioned typed assertion schema used by immutable Case versions.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(
    tag = "kind",
    rename_all = "snake_case",
    rename_all_fields = "camelCase",
    deny_unknown_fields
)]
pub enum TypedAssertion {
    OutputContains {
        turn: usize,
        values: Vec<String>,
        #[serde(default)]
        any: bool,
    },
    WorkspaceFile {
        path: String,
        #[serde(default)]
        contains: Vec<String>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_bytes: Option<usize>,
    },
    CommandExit {
        name: String,
        command_ref: String,
        expected_exit_code: i32,
    },
    TraceEvent {
        event_type: String,
        #[serde(default)]
        minimum_count: usize,
    },
    ArtifactExists {
        artifact_kind: String,
    },
    /// A custom evaluator that is executed by the isolated executor. The
    /// control plane accepts only an attested result for the exact immutable
    /// plugin reference; it never loads plugin code into the service process.
    Plugin {
        plugin_ref: String,
        #[serde(default)]
        config: Value,
    },
}

/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct VersionedResourceRef {
    pub id: String,
    pub version: String,
}

/// Agent is an optimistic precondition; Runtime ID remains the execution
/// selector and Runtime is the authoritative owner of their binding.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluationSubject {
    pub agent_id: String,
    pub runtime_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterEnvironmentProfileRequest {
    pub profile_id: String,
    pub version: String,
    /// Existing Workspace Environment Space selected for the Task. Provisioned
    /// environments can be added later behind the same immutable profile.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub space_id: Option<String>,
    #[serde(default)]
    pub metadata: std::collections::BTreeMap<String, String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EnvironmentProfileVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub profile_id: String,
    pub version: String,
    pub digest: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub space_id: Option<String>,
    #[serde(default)]
    pub metadata: std::collections::BTreeMap<String, String>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvaluatorImplementation {
    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
    BuiltinAssertions { assertions: Vec<TypedAssertion> },
    /// Model-graded evaluator executed by the isolated executor. The control
    /// plane only maps its attested evidence onto a verdict.
    LlmJudge {
        /// Pinned judge model. Changing it requires a new evaluator version so
        /// judge drift can never be mistaken for subject regression.
        model_ref: String,
        /// Digest of the rubric prompt handed to the judge.
        prompt_digest: String,
        #[serde(default)]
        max_cost_micros: u64,
    },
}

impl EvaluatorImplementation {
    pub const fn is_model_graded(&self) -> bool {
        matches!(self, Self::LlmJudge { .. })
    }
}

/// Human-label agreement accumulated for one evaluator id.
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct JudgeCalibration {
    pub labeled_samples: u64,
    pub agreed_samples: u64,
}

impl JudgeCalibration {
    pub fn agreement(&self) -> f64 {
        if self.labeled_samples == 0 {
            0.0
        } else {
            self.agreed_samples as f64 / self.labeled_samples as f64
        }
    }
}

/// Minimum human-label evidence before a model-graded evaluator may gate.
pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterEvaluatorVersionRequest {
    pub evaluator_id: String,
    pub version: String,
    pub implementation: EvaluatorImplementation,
    #[serde(default)]
    pub metric: MetricSpec,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluatorVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub evaluator_id: String,
    pub version: String,
    pub digest: String,
    pub implementation: EvaluatorImplementation,
    #[serde(default)]
    pub metric: MetricSpec,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvaluatorVerdict {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub passed: bool,
    #[serde(default)]
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    /// Judge-side cost, kept apart from the subject's execution cost.
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub assertions: Vec<AssertionVerdict>,
}

/// Immutable Case content. A Case is the node that owns the evaluation
/// dependency edges, so it is registered and versioned in its own right and a
/// Suite only references it.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterEvalCaseRequest {
    pub case_id: String,
    pub version: String,
    pub category: String,
    #[serde(default)]
    pub turns: Vec<EvalTurn>,
    /// Small, immutable text fixtures materialised into the selected Space
    /// before Runtime executes the case. Large fixtures remain external and
    /// are referenced through `fixtureRef`.
    #[serde(default)]
    pub workspace_files: std::collections::BTreeMap<String, String>,
    #[serde(default)]
    pub assertions: Vec<TypedAssertion>,
    /// Versioned environment selection.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<VersionedResourceRef>,
    /// Domain-owned evaluator policies executed after evidence collection.
    #[serde(default)]
    pub evaluator_versions: Vec<VersionedResourceRef>,
    #[serde(default)]
    pub verify_commands: Vec<VerifyCommand>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub fixture_ref: Option<String>,
    #[serde(default)]
    pub budget: EvalBudget,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalCaseVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub case_id: String,
    pub version: String,
    pub digest: String,
    pub category: String,
    #[serde(default)]
    pub turns: Vec<EvalTurn>,
    #[serde(default)]
    pub workspace_files: std::collections::BTreeMap<String, String>,
    #[serde(default)]
    pub assertions: Vec<TypedAssertion>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<VersionedResourceRef>,
    #[serde(default)]
    pub evaluator_versions: Vec<VersionedResourceRef>,
    #[serde(default)]
    pub verify_commands: Vec<VerifyCommand>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub fixture_ref: Option<String>,
    #[serde(default)]
    pub budget: EvalBudget,
    pub created_at_ms: i64,
}

impl EvalCaseVersion {
    pub fn reference(&self) -> VersionedResourceRef {
        VersionedResourceRef {
            id: self.case_id.clone(),
            version: self.version.clone(),
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalBudget {
    pub timeout_ms: u64,
    pub max_output_bytes: usize,
    pub max_cost_micros: u64,
}

impl Default for EvalBudget {
    fn default() -> Self {
        Self {
            timeout_ms: 300_000,
            max_output_bytes: 8 * 1024 * 1024,
            max_cost_micros: 0,
        }
    }
}

/// Ordered selection of immutable Case versions. A Suite adds no content of
/// its own, so changing one Case never rewrites the others.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalSuiteVersion {
    pub tenant_id: String,
    pub project_id: String,
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub cases: Vec<VersionedResourceRef>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RegisterSuiteRequest {
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub cases: Vec<VersionedResourceRef>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OperationState {
    Queued,
    Running,
    Succeeded,
    Failed,
    Canceling,
    Canceled,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CaseRunState {
    Queued,
    Running,
    Passed,
    QualityFailed,
    InfraFailed,
    InvalidCase,
    Canceled,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CreateEvalRunRequest {
    /// Suite version to expand. Mutually exclusive with `cases`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub suite: Option<VersionedResourceRef>,
    /// Ad-hoc Case selection. Because Case is a catalog resource, a run can be
    /// composed without first inventing a Suite for it.
    #[serde(default)]
    pub cases: Vec<VersionedResourceRef>,
    pub subject_version: String,
    pub executor_profile: String,
    /// Concrete execution target. New Runtime Task based runs require it;
    /// subject_version remains as a compatibility/report label.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default)]
    pub seed: u64,
    #[serde(default = "default_concurrency")]
    pub concurrency: usize,
    #[serde(default)]
    pub idempotency_key: String,
    /// Independent deterministic samples per `(case, variant)`.
    #[serde(default = "default_samples")]
    pub samples: u32,
    /// Empty preserves the single-subject run. A non-empty matrix expands
    /// the run without requiring consumers to coordinate many operations.
    #[serde(default)]
    pub experiment_matrix: Vec<ExperimentVariant>,
}

fn default_concurrency() -> usize {
    8
}

fn default_samples() -> u32 {
    1
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ExperimentVariant {
    pub id: String,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default)]
    pub parameters: Value,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalOperation {
    pub id: String,
    pub run_id: String,
    pub state: OperationState,
    pub phase: String,
    pub attempt: u32,
    pub completed_cases: usize,
    pub total_cases: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub report_id: Option<String>,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
    pub version: i64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_poll_after_ms: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error_code: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetOperationRequest {
    pub operation_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CancelRunRequest {
    pub run_id: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListCaseRunsRequest {
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<CaseRunState>,
    #[serde(default = "default_case_page")]
    pub limit: usize,
}

fn default_case_page() -> usize {
    50
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunSummary {
    pub id: String,
    pub run_id: String,
    pub case_id: String,
    pub case_version: String,
    pub position: usize,
    pub state: CaseRunState,
    pub attempt: u32,
    pub duration_ms: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    #[serde(default)]
    pub evidence_refs: Vec<EvidenceRef>,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseRunPage {
    pub items: Vec<CaseRunSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvalAccessScope {
    pub tenant_id: String,
    pub project_id: String,
    pub subject: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxExecutionRequest {
    /// Tenant/project ownership is passed explicitly so the isolated executor
    /// can only finalize evidence in the namespace later validated by the
    /// control plane.
    pub scope: EvalAccessScope,
    pub operation_id: String,
    pub run_id: String,
    pub case_run_id: String,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<EnvironmentProfileVersion>,
    #[serde(default)]
    pub evaluator_versions: Vec<EvaluatorVersion>,
    pub seed: u64,
    pub case: EvalCaseVersion,
    pub fencing_token: i64,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant: Option<ExperimentVariant>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct PluginEvaluationEvidence {
    pub plugin_ref: String,
    pub passed: bool,
    #[serde(default)]
    pub detail: String,
    /// Attestation produced by the configured sandbox boundary. Production
    /// evaluation fails closed when it is absent.
    pub attestation: String,
}

/// Attested outcome of a model-graded evaluator. Produced by the isolated
/// executor; the control plane never calls a judge model itself.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeEvaluationEvidence {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub passed: bool,
    /// Judge model actually used, checked against the pinned reference.
    pub model_ref: String,
    #[serde(default)]
    pub detail: String,
    #[serde(default)]
    pub cost_micros: u64,
    pub attestation: String,
}

/// Immutable evidence object owned by the scoped Evaluate namespace.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvidenceRef {
    pub reference: String,
    pub tenant_id: String,
    pub project_id: String,
    pub run_id: String,
    pub case_run_id: String,
    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
    pub digest: String,
    pub size_bytes: u64,
    /// Attestation bound to the isolated executor response boundary.
    pub attestation: String,
}

/// Bounded evidence returned by the isolated executor. Verdict calculation
/// remains owned by Evaluate and is deterministic from this typed structure.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SandboxExecutionEvidence {
    #[serde(default)]
    pub responses: Vec<String>,
    #[serde(default)]
    pub workspace_files: std::collections::BTreeMap<String, String>,
    #[serde(default)]
    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
    #[serde(default)]
    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
    #[serde(default)]
    pub artifact_kinds: Vec<String>,
    /// Immutable blob/object references owned by Evaluate's evidence namespace.
    #[serde(default)]
    pub evidence_refs: Vec<EvidenceRef>,
    #[serde(default)]
    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
    #[serde(default)]
    pub judge_evaluations: Vec<JudgeEvaluationEvidence>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub infrastructure_error: Option<String>,
    /// A sandbox may explicitly reject an immutable case as unsupported or
    /// invalid. This is deliberately typed so provider text is never parsed as
    /// a terminal-state code.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub invalid_case_error: Option<String>,
    #[serde(default)]
    pub duration_ms: u64,
    /// Executor-measured cost in millionths of the configured billing unit.
    #[serde(default)]
    pub cost_micros: u64,
}

/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
/// never needs a callback URL for the Runtime.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimEvaluationCaseRequest {
    pub evaluate_task_id: String,
    #[serde(default = "default_driver_lease_ms")]
    pub lease_ms: i64,
}

fn default_driver_lease_ms() -> i64 {
    60_000
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EvaluationCaseLease {
    pub execution: SandboxExecutionRequest,
    pub lease_until_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimEvaluationCaseResponse {
    pub operation: EvalOperation,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub case: Option<EvaluationCaseLease>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewEvaluationCaseRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    #[serde(default = "default_driver_lease_ms")]
    pub lease_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RenewEvaluationCaseResponse {
    pub renewed: bool,
    pub lease_until_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvaluationEvidenceRequest {
    pub evaluate_task_id: String,
    pub case_run_id: String,
    pub fencing_token: i64,
    pub evidence: SandboxExecutionEvidence,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitEvaluationEvidenceResponse {
    pub accepted: bool,
    pub operation: EvalOperation,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetReportRequest {
    pub report_id: String,
}

// ---------------------------------------------------------------------------
// Evaluate administration query model
// ---------------------------------------------------------------------------

pub const DEFAULT_ADMIN_PAGE_SIZE: usize = 50;
pub const MAX_ADMIN_PAGE_SIZE: usize = 200;

fn default_admin_page() -> usize {
    DEFAULT_ADMIN_PAGE_SIZE
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminCasesRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminCasesRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminSuitesRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminSuitesRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminRunsRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<OperationState>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminRunsRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            state: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListAdminReviewsRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state: Option<ReviewState>,
    #[serde(default = "default_admin_page")]
    pub limit: usize,
}

impl Default for ListAdminReviewsRequest {
    fn default() -> Self {
        Self {
            cursor: None,
            state: None,
            limit: DEFAULT_ADMIN_PAGE_SIZE,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvaluateAdminOverview {
    pub case_versions: u64,
    pub suite_versions: u64,
    pub run_states: std::collections::BTreeMap<String, u64>,
    pub case_states: std::collections::BTreeMap<String, u64>,
    pub review_states: std::collections::BTreeMap<String, u64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminCaseSummary {
    pub case_id: String,
    pub version: String,
    pub digest: String,
    pub category: String,
    pub turns: usize,
    pub evaluator_versions: Vec<VersionedResourceRef>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub environment_profile: Option<VersionedResourceRef>,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminCasePage {
    pub items: Vec<AdminCaseSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminSuiteSummary {
    pub suite_id: String,
    pub version: String,
    pub manifest_digest: String,
    pub schema_version: u32,
    pub case_count: usize,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminSuitePage {
    pub items: Vec<AdminSuiteSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminRunSummary {
    pub run_id: String,
    pub operation_id: String,
    /// Present only when the run was expanded from a Suite version.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub suite: Option<VersionedResourceRef>,
    pub subject_version: String,
    pub executor_profile: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub subject: Option<EvaluationSubject>,
    pub state: OperationState,
    pub phase: String,
    pub completed_cases: usize,
    pub total_cases: usize,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub report_id: Option<String>,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
    pub version: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminRunPage {
    pub items: Vec<AdminRunSummary>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AdminReviewPage {
    pub items: Vec<HumanReview>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub next_cursor: Option<String>,
}

/// Error body shared by every non-2xx response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApiError {
    pub status: u16,
    pub message: String,
}

#[cfg(test)]
mod tests {
    use super::{
        DEFAULT_ADMIN_PAGE_SIZE, ListAdminCasesRequest, ListAdminReviewsRequest,
        ListAdminRunsRequest, ListAdminSuitesRequest, TypedAssertion,
    };
    use serde_json::json;

    #[test]
    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
        let assertion = TypedAssertion::TraceEvent {
            event_type: "task.completed".into(),
            minimum_count: 1,
        };
        let value = serde_json::to_value(&assertion).unwrap();
        assert_eq!(
            value,
            json!({
                "kind": "trace_event",
                "eventType": "task.completed",
                "minimumCount": 1
            })
        );
        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
    }

    #[test]
    fn admin_queries_use_the_same_bounded_default_in_rust_and_serde() {
        assert_eq!(
            ListAdminCasesRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        assert_eq!(
            ListAdminSuitesRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        assert_eq!(
            ListAdminRunsRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        assert_eq!(
            ListAdminReviewsRequest::default().limit,
            DEFAULT_ADMIN_PAGE_SIZE
        );
        let decoded: ListAdminRunsRequest = serde_json::from_value(json!({})).unwrap();
        assert_eq!(decoded.limit, DEFAULT_ADMIN_PAGE_SIZE);
    }
}