agent-evaluate-v2-contract 0.1.0

//! Verdicts, reports and the release gate.

use std::collections::BTreeMap;

use serde::{Deserialize, Serialize};

use crate::catalog::{Capability, SnapshotRef};
use crate::evaluator::{MetricClass, MetricDecision};
use crate::run::CaseRunState;

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AssertionVerdict {
    pub rule: String,
    pub passed: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub detail: Option<String>,
}

/// A measured value with the bound it was judged against. Values are stored
/// even when they cannot fail: trace retention is shorter than conclusion
/// retention, so a report can never rely on re-reading the trace later.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricValue {
    pub metric_id: String,
    pub value: f64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub min: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub max: Option<f64>,
    pub passed: bool,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub detail: Option<String>,
}

/// Why a judged verdict is not allowed to gate this time. Recording the
/// reason keeps "the judge was ignored" from looking like "the judge passed".
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum VerdictDowngrade {
    /// Inter-judge agreement fell below the evaluator's threshold.
    LowAgreement,
    /// A judge sharing the subject's model had its vote discarded.
    SelfGraded,
    /// The ensemble stopped early on its cost budget.
    PartialEnsemble,
    /// The judge model is not on the platform gate allowlist.
    JudgeNotAllowlisted,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvaluatorVerdict {
    pub evaluator_id: String,
    pub evaluator_digest: String,
    pub kind: String,
    pub passed: bool,
    #[serde(default)]
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    #[serde(default)]
    pub assertions: Vec<AssertionVerdict>,
    #[serde(default)]
    pub metrics: Vec<MetricValue>,
    /// Judge spend, reported apart from the subject's execution cost.
    #[serde(default)]
    pub judge_cost_micros: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub agreement: Option<f64>,
    #[serde(default)]
    pub downgrades: Vec<VerdictDowngrade>,
}

impl EvaluatorVerdict {
    /// A verdict gates only when its class allows it and nothing forced a
    /// downgrade.
    pub fn gates(&self) -> bool {
        self.class.affects_case_verdict() && self.downgrades.is_empty()
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CaseResult {
    pub case_run_id: String,
    #[serde(rename = "snapshotRef")]
    pub snapshot_ref: SnapshotRef,
    pub case_id: String,
    pub case_digest: String,
    pub category: String,
    pub state: CaseRunState,
    pub passed: bool,
    #[serde(default)]
    pub turns_completed: usize,
    #[serde(default)]
    pub verdicts: Vec<EvaluatorVerdict>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    #[serde(default)]
    pub duration_ms: u64,
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub sample_index: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub variant_id: Option<String>,
}

/// One observed slice of a metric: successes over a bounded denominator.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RateSlice {
    pub observed: u64,
    pub passed: u64,
}

impl RateSlice {
    pub fn rate(&self) -> f64 {
        if self.observed == 0 {
            0.0
        } else {
            self.passed as f64 / self.observed as f64
        }
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricSummary {
    pub evaluator_id: String,
    pub evaluator_digest: String,
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    pub observed: u64,
    pub passed: u64,
    pub rate: f64,
    /// Wilson interval: suites are commonly 30-50 cases, where the normal
    /// approximation is unusable near 0 and 1.
    pub wilson_low: f64,
    pub wilson_high: f64,
    #[serde(default)]
    pub judge_cost_micros: u64,
    #[serde(default)]
    pub by_category: BTreeMap<String, RateSlice>,
    #[serde(default)]
    pub by_variant: BTreeMap<String, RateSlice>,
}

/// Reports are revisioned: revision 1 seals once synchronous metrics are in,
/// and each completed batch of asynchronous analysis appends a new one.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct EvalReport {
    pub id: String,
    pub run_id: String,
    pub revision: u32,
    #[serde(default)]
    pub snapshots: Vec<SnapshotRef>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub capability: Option<Capability>,
    pub total_cases: usize,
    pub passed_cases: usize,
    pub quality_failed_cases: usize,
    pub infra_failed_cases: usize,
    pub invalid_cases: usize,
    pub analysis_failed_cases: usize,
    /// Pass rate over cases that produced a quality signal at all.
    pub effective_pass_rate: f64,
    #[serde(default)]
    pub metrics: Vec<MetricSummary>,
    #[serde(default)]
    pub case_results: Vec<CaseResult>,
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub judge_cost_micros: u64,
    #[serde(default)]
    pub duration_ms: u64,
    pub sealed_at_ms: i64,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GetReportRequest {
    pub run_id: String,
    /// Absent means the latest sealed revision.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub revision: Option<u32>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct MetricThreshold {
    pub evaluator_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub minimum_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub maximum_regression: Option<f64>,
}

/// Declarative gate: guardrails are absolute, goals are statistical, and
/// diagnostics never gate.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct GatePolicy {
    #[serde(default)]
    pub minimum_pass_rate: f64,
    #[serde(default = "default_max_infra_rate")]
    pub maximum_infra_failure_rate: f64,
    #[serde(default = "default_significance")]
    pub significance_level: f64,
    /// Regression tolerated on a goal metric before it can block.
    #[serde(default)]
    pub minimum_effect: f64,
    #[serde(default)]
    pub goal_minimum_rate: f64,
    #[serde(default)]
    pub overrides: Vec<MetricThreshold>,
}

fn default_max_infra_rate() -> f64 {
    1.0
}

fn default_significance() -> f64 {
    0.05
}

impl Default for GatePolicy {
    fn default() -> Self {
        Self {
            minimum_pass_rate: 0.0,
            maximum_infra_failure_rate: default_max_infra_rate(),
            significance_level: default_significance(),
            minimum_effect: 0.0,
            goal_minimum_rate: 0.0,
            overrides: Vec::new(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CompareReportsRequest {
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub revision: Option<u32>,
    pub baseline_run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub baseline_revision: Option<u32>,
    #[serde(default)]
    pub policy: GatePolicy,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GateViolation {
    pub evaluator_id: String,
    pub class: MetricClass,
    pub decision: MetricDecision,
    pub reason: String,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GateDecision {
    pub allowed: bool,
    #[serde(default)]
    pub violations: Vec<GateViolation>,
}