agent-evaluate-v2-contract 0.1.0

//! Evaluators: the domain rules attached to a case.
//!
//! An evaluator is inlined on its case because it is only meaningful for that
//! case. Cross-snapshot comparability comes from `(evaluatorId, digest)`, not
//! from a separate catalog resource nobody remembers to version.

use serde::{Deserialize, Serialize};
use serde_json::Value;

/// How a metric participates in release decisions.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricClass {
    /// Quality we actively push up. Regressions gate on statistical evidence.
    Goal,
    /// Must never break. A single failure gates without a significance test.
    Guardrail,
    /// Observed only. Never contributes to a case verdict or a gate.
    #[default]
    Diagnostic,
}

impl MetricClass {
    pub const fn affects_case_verdict(self) -> bool {
        matches!(self, Self::Goal | Self::Guardrail)
    }

    pub const fn as_str(self) -> &'static str {
        match self {
            Self::Goal => "goal",
            Self::Guardrail => "guardrail",
            Self::Diagnostic => "diagnostic",
        }
    }
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricDecision {
    BlockRelease,
    RollBack,
    Investigate,
    #[default]
    ObserveOnly,
}

impl MetricDecision {
    pub const fn is_blocking(self) -> bool {
        matches!(self, Self::BlockRelease | Self::RollBack)
    }
}

#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct MetricSpec {
    #[serde(default)]
    pub class: MetricClass,
    #[serde(default)]
    pub decision: MetricDecision,
    /// Contacted when the metric is recommended for retirement.
    #[serde(default)]
    pub owner: String,
    /// Guardrails leave this unset: staying green forever is their job.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub retire_after_idle_runs: Option<u32>,
}

/// Deterministic assertion over bounded evidence.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(
    tag = "kind",
    rename_all = "snake_case",
    rename_all_fields = "camelCase",
    deny_unknown_fields
)]
pub enum TypedAssertion {
    OutputContains {
        turn: usize,
        values: Vec<String>,
        #[serde(default)]
        any: bool,
    },
    WorkspaceFile {
        path: String,
        #[serde(default)]
        contains: Vec<String>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max_bytes: Option<usize>,
    },
    CommandExit {
        name: String,
        expected_exit_code: i32,
    },
    ArtifactExists {
        artifact_kind: String,
    },
}

/// Deterministic metric computed from the case's trace (plus evidence).
///
/// Bounds are inclusive. A metric with neither bound set is observational: it
/// records a value without ever failing, which is a legitimate way to collect
/// a signal before deciding what "good" means.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(
    tag = "kind",
    rename_all = "snake_case",
    rename_all_fields = "camelCase",
    deny_unknown_fields
)]
pub enum TraceMetricSpec {
    EventCount {
        event_type: String,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        min: Option<u64>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    ToolCallCount {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        tool: Option<String>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        min: Option<u64>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    ToolErrorRate {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<f64>,
    },
    TurnCount {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        min: Option<u64>,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    LatencyMs {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    TokenCostMicros {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    RetryCount {
        #[serde(default, skip_serializing_if = "Option::is_none")]
        max: Option<u64>,
    },
    /// Same `(tool, argsDigest)` repeated more than `max_repeats` in a row.
    LoopDetected { max_repeats: u64 },
    /// Listed event types must appear in this relative order.
    EventSequence { event_types: Vec<String> },
}

impl TraceMetricSpec {
    /// Stable id used as the metric's storage key within one evaluator.
    pub fn metric_id(&self) -> String {
        match self {
            Self::EventCount { event_type, .. } => format!("event_count:{event_type}"),
            Self::ToolCallCount { tool, .. } => match tool {
                Some(tool) => format!("tool_call_count:{tool}"),
                None => "tool_call_count".to_string(),
            },
            Self::ToolErrorRate { .. } => "tool_error_rate".to_string(),
            Self::TurnCount { .. } => "turn_count".to_string(),
            Self::LatencyMs { .. } => "latency_ms".to_string(),
            Self::TokenCostMicros { .. } => "token_cost_micros".to_string(),
            Self::RetryCount { .. } => "retry_count".to_string(),
            Self::LoopDetected { .. } => "loop_detected".to_string(),
            Self::EventSequence { .. } => "event_sequence".to_string(),
        }
    }

    /// True when the metric needs authoritative tool/retry/token data, which
    /// only a runtime-written trace can provide.
    pub const fn requires_runtime_trace(&self) -> bool {
        matches!(
            self,
            Self::ToolCallCount { .. }
                | Self::ToolErrorRate { .. }
                | Self::TokenCostMicros { .. }
                | Self::RetryCount { .. }
                | Self::LoopDetected { .. }
                | Self::EventSequence { .. }
                | Self::EventCount { .. }
        )
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JudgeKind {
    /// Absolute score against a rubric.
    Rubric,
    /// Compared against a baseline output.
    Pairwise,
    /// Compared against a reference answer.
    Reference,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JudgeAggregation {
    /// Median of the per-judge scores. Robust to one outlier judge.
    Median,
    WeightedMean,
    /// Majority vote; requires an odd number of at least three judges.
    Majority,
    /// Everyone must agree; requires at least two judges.
    Unanimous,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeModel {
    pub model_ref: String,
    #[serde(default = "default_weight")]
    pub weight: f64,
}

fn default_weight() -> f64 {
    1.0
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RubricCriterion {
    pub id: String,
    pub weight: f64,
}

#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RubricScale {
    pub min: f64,
    pub max: f64,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Rubric {
    pub criteria: Vec<RubricCriterion>,
    pub scale: RubricScale,
}

/// What a judge is allowed to look at. Anything requiring a trace summary
/// forces the case to demand a runtime-level trace.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeInputs {
    #[serde(default)]
    pub output: bool,
    #[serde(default)]
    pub trace_summary: bool,
    #[serde(default)]
    pub workspace_files: Vec<String>,
}

/// Model-graded evaluator. Everything that can move a verdict is pinned and
/// digested, so judge drift can never be mistaken for subject regression.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct LlmJudgeSpec {
    pub judge: JudgeKind,
    /// One entry is a single judge; more than one is a cross-model ensemble.
    pub judges: Vec<JudgeModel>,
    pub aggregation: JudgeAggregation,
    /// Inter-judge agreement below this marks the result `low_agreement`,
    /// which drops it out of gating and into the human review queue.
    #[serde(default = "default_min_agreement")]
    pub min_agreement: f64,
    pub prompt_digest: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub rubric: Option<Rubric>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub pass_threshold: Option<f64>,
    /// Self-consistency samples per judge, aggregated within the judge first.
    #[serde(default = "default_samples")]
    pub samples: u32,
    #[serde(default)]
    pub temperature: f64,
    /// Pairwise only: run both orderings to cancel position bias.
    #[serde(default)]
    pub position_swap: bool,
    /// Reference only: pointer to the expected answer.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub reference_ref: Option<String>,
    /// Budget for the whole ensemble, not per judge.
    #[serde(default)]
    pub max_cost_micros: u64,
    #[serde(default)]
    pub inputs: JudgeInputs,
}

fn default_min_agreement() -> f64 {
    0.66
}

fn default_samples() -> u32 {
    1
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvaluatorImplementation {
    /// Deterministic assertions over bounded evidence.
    Assertions { assertions: Vec<TypedAssertion> },
    /// Deterministic metrics derived from the case's trace.
    TraceMetrics { metrics: Vec<TraceMetricSpec> },
    /// Model-graded, executed by the isolated executor.
    LlmJudge(LlmJudgeSpec),
    /// Custom evaluator executed by the isolated executor; the control plane
    /// only maps an attested result and never loads plugin code.
    Plugin {
        plugin_ref: String,
        #[serde(default)]
        config: Value,
    },
}

impl EvaluatorImplementation {
    pub const fn is_asynchronous(&self) -> bool {
        matches!(self, Self::LlmJudge(_) | Self::Plugin { .. })
    }

    pub const fn kind_str(&self) -> &'static str {
        match self {
            Self::Assertions { .. } => "assertions",
            Self::TraceMetrics { .. } => "trace_metrics",
            Self::LlmJudge(_) => "llm_judge",
            Self::Plugin { .. } => "plugin",
        }
    }
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CaseEvaluator {
    pub evaluator_id: String,
    /// Server-computed. A client-supplied value is recomputed and compared.
    #[serde(default)]
    pub digest: String,
    #[serde(default)]
    pub metric: MetricSpec,
    pub implementation: EvaluatorImplementation,
}

/// Minimum human-label evidence before a model-graded evaluator may gate.
/// Applied to the aggregated ensemble output, not to an individual judge.
pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;

/// Platform-maintained judge catalogue entry. Only models on this list may
/// back a gating evaluator; a tenant may still judge with any visible model
/// as long as the metric stays diagnostic.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeAllowlistEntry {
    pub model_ref: String,
    /// False while the model is still accumulating calibration evidence.
    #[serde(default)]
    pub gate_approved: bool,
    #[serde(default)]
    pub labeled_samples: u64,
    #[serde(default)]
    pub human_agreement: f64,
    #[serde(default)]
    pub cost_tier: String,
    #[serde(default)]
    pub notes: String,
    pub updated_at_ms: i64,
}

#[cfg(test)]
mod tests {
    use super::{MetricClass, TraceMetricSpec, TypedAssertion};
    use serde_json::json;

    #[test]
    fn assertions_use_snake_case_tags_and_camel_case_fields() {
        let value = serde_json::to_value(TypedAssertion::OutputContains {
            turn: 0,
            values: vec!["ok".into()],
            any: false,
        })
        .unwrap();
        assert_eq!(
            value,
            json!({ "kind": "output_contains", "turn": 0, "values": ["ok"], "any": false })
        );
    }

    #[test]
    fn trace_metric_ids_are_stable_and_distinguish_per_tool_slices() {
        let all = TraceMetricSpec::ToolCallCount {
            tool: None,
            min: None,
            max: None,
        };
        let one = TraceMetricSpec::ToolCallCount {
            tool: Some("bash".into()),
            min: None,
            max: None,
        };
        assert_eq!(all.metric_id(), "tool_call_count");
        assert_eq!(one.metric_id(), "tool_call_count:bash");
    }

    #[test]
    fn diagnostic_is_the_default_class_so_new_signals_cannot_gate_by_accident() {
        assert_eq!(MetricClass::default(), MetricClass::Diagnostic);
        assert!(!MetricClass::Diagnostic.affects_case_verdict());
    }
}