agent-evaluate-contract 0.5.0

//! Metric taxonomy, report slices and metric health.
//!
//! Infra owns the taxonomy, its gate semantics and the statistics; the domain
//! owns which class a given evaluator belongs to and whether to retire it.

use serde::{Deserialize, Serialize};

/// How a metric participates in release decisions.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricClass {
    /// Quality we actively push up. Regressions gate on statistical evidence.
    #[default]
    Goal,
    /// Must never break. A single failure gates without a significance test.
    Guardrail,
    /// Observed only. Never contributes to `EvalCaseResult::passed` or a gate.
    Diagnostic,
}

impl MetricClass {
    /// Diagnostic metrics are deliberately excluded from case verdicts so a
    /// team can observe a new signal without making CI red.
    pub const fn affects_case_verdict(self) -> bool {
        matches!(self, Self::Goal | Self::Guardrail)
    }

    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Goal => "goal",
            Self::Guardrail => "guardrail",
            Self::Diagnostic => "diagnostic",
        }
    }
}

/// The action a moving metric triggers. A metric with no action is noise.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricDecision {
    #[default]
    BlockRelease,
    RollBack,
    Investigate,
    ObserveOnly,
}

impl MetricDecision {
    pub const fn is_blocking(self) -> bool {
        matches!(self, Self::BlockRelease | Self::RollBack)
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct MetricSpec {
    #[serde(default)]
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    /// Contacted when the metric is recommended for retirement.
    #[serde(default)]
    pub owner: String,
    /// Guardrails leave this unset: staying green forever is their job.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub retire_after_idle_runs: Option<u32>,
}

impl Default for MetricSpec {
    /// Legacy evaluators registered before the taxonomy existed keep their
    /// blocking behaviour.
    fn default() -> Self {
        Self {
            class: MetricClass::Goal,
            decision: MetricDecision::BlockRelease,
            owner: String::new(),
            retire_after_idle_runs: None,
        }
    }
}

/// Human-label agreement accumulated for one evaluator id.
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct JudgeCalibration {
    pub labeled_samples: u64,
    pub agreed_samples: u64,
}

impl JudgeCalibration {
    pub fn agreement(&self) -> f64 {
        if self.labeled_samples == 0 {
            0.0
        } else {
            self.agreed_samples as f64 / self.labeled_samples as f64
        }
    }
}

/// Minimum human-label evidence before a model-graded evaluator may gate.
pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;

/// One observed slice of a metric: successes over a bounded denominator.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RateSlice {
    pub observed: u64,
    pub passed: u64,
}

impl RateSlice {
    pub fn rate(&self) -> f64 {
        if self.observed == 0 {
            0.0
        } else {
            self.passed as f64 / self.observed as f64
        }
    }
}

/// Aggregated outcome of one evaluator version within a single report.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricSummary {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    pub observed: u64,
    pub passed: u64,
    pub rate: f64,
    /// Wilson score interval. Suites are commonly 30-50 cases, where the
    /// normal approximation is unusable near 0 and 1.
    pub wilson_low: f64,
    pub wilson_high: f64,
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub by_category: std::collections::BTreeMap<String, RateSlice>,
    #[serde(default)]
    pub by_variant: std::collections::BTreeMap<String, RateSlice>,
}

// ---------------------------------------------------------------------------
// Candidate metrics
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ListMetricCandidatesRequest {
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub run_id: Option<String>,
    /// Labels seen fewer times than this are ignored as anecdotes.
    #[serde(default = "default_min_occurrences")]
    pub minimum_occurrences: u64,
}

fn default_min_occurrences() -> u64 {
    3
}

/// A labelled failure mode and whether it actually predicts the outcome the
/// system exists to decide.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricCandidate {
    pub label: String,
    pub occurrences: u64,
    pub failed_occurrences: u64,
    /// Phi coefficient between the label and a failing human outcome.
    pub correlation: f64,
    /// False when the label is frequent but uncorrelated with the decision.
    pub recommended: bool,
    pub rationale: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricCandidateReport {
    pub labeled_reviews: u64,
    pub failed_reviews: u64,
    pub candidates: Vec<MetricCandidate>,
}

// ---------------------------------------------------------------------------
// Metric health
// ---------------------------------------------------------------------------

fn default_health_window() -> u32 {
    100
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct MetricHealthRequest {
    /// Most recent runs inspected per evaluator version.
    #[serde(default = "default_health_window")]
    pub window_runs: u32,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub evaluator_id: Option<String>,
}

impl Default for MetricHealthRequest {
    fn default() -> Self {
        Self {
            window_runs: default_health_window(),
            evaluator_id: None,
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricRecommendation {
    Keep,
    Retire,
    Investigate,
}

/// Per-run observation of one evaluator version. Written once when a run's
/// report is sealed, so health queries never rescan case results.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricObservation {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub class: MetricClass,
    pub run_id: String,
    pub observed: u64,
    pub passed: u64,
    pub cost_micros: u64,
    /// Variants compared in the run that produced this observation.
    pub variant_count: u64,
    /// True when the metric separated at least two variants of that run.
    pub discriminated: bool,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricHealth {
    pub evaluator_id: String,
    pub evaluator_version: String,
    pub class: MetricClass,
    pub runs: u64,
    pub observed: u64,
    pub passed: u64,
    pub rate: f64,
    /// Consecutive most-recent runs without a single failure.
    pub idle_runs: u64,
    /// Runs where the metric separated variants, over runs that had variants.
    pub discrimination: f64,
    pub discrimination_opportunities: u64,
    pub cost_micros_per_run: u64,
    pub labeled_samples: u64,
    pub human_agreement: f64,
    pub recommendation: MetricRecommendation,
    pub rationale: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetricHealthReport {
    pub metrics: Vec<MetricHealth>,
}