agentics_domain/models/challenge/
metrics.rs

1use serde::{Deserialize, Serialize};
2
3use super::super::names::MetricName;
4
5/// Whether a metric is better when it is larger or smaller.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
7#[serde(rename_all = "snake_case")]
8pub enum MetricDirection {
9    Maximize,
10    Minimize,
11}
12
13/// Visibility level for a metric emitted by the evaluator.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
15#[serde(rename_all = "snake_case")]
16pub enum MetricVisibility {
17    /// Visible in validation feedback and official result views.
18    Public,
19    /// Visible only after a ranking-visible official evaluation.
20    Official,
21}
22
23/// One metric that an evaluator may emit in aggregate or per-run result payloads.
24#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
25pub struct MetricDefinitionSpec {
26    pub name: MetricName,
27    pub label: String,
28    #[serde(default, skip_serializing_if = "Option::is_none")]
29    pub unit: Option<String>,
30    pub direction: MetricDirection,
31    pub visibility: MetricVisibility,
32    #[serde(default, skip_serializing_if = "Option::is_none")]
33    pub metric_description: Option<String>,
34}
35
36/// Ranking configuration for a challenge.
37#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
38pub struct RankingSpec {
39    pub primary_metric_name: MetricName,
40    #[serde(default)]
41    #[schemars(required)]
42    pub tie_breaker_metric_names: Vec<MetricName>,
43}
44
45/// Metric schema embedded in `spec.json`.
46#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
47pub struct MetricSchemaSpec {
48    pub metrics: Vec<MetricDefinitionSpec>,
49    pub ranking: RankingSpec,
50}
51
52impl MetricSchemaSpec {
53    /// Look up a metric definition by name.
54    pub fn metric(&self, metric_name: &MetricName) -> Option<&MetricDefinitionSpec> {
55        self.metrics
56            .iter()
57            .find(|metric| &metric.name == metric_name)
58    }
59
60    /// Primary ranking metric declared by this challenge.
61    pub fn primary_metric(&self) -> Option<&MetricDefinitionSpec> {
62        self.metric(&self.ranking.primary_metric_name)
63    }
64}
65
66impl Default for MetricSchemaSpec {
67    /// Handles default for this module.
68    fn default() -> Self {
69        Self {
70            metrics: vec![MetricDefinitionSpec {
71                name: MetricName::score(),
72                label: "Score".to_string(),
73                unit: None,
74                direction: MetricDirection::Maximize,
75                visibility: MetricVisibility::Public,
76                metric_description: Some("Challenge-defined compatibility score.".to_string()),
77            }],
78            ranking: RankingSpec {
79                primary_metric_name: MetricName::score(),
80                tie_breaker_metric_names: vec![],
81            },
82        }
83    }
84}
agentics_domain/models/challenge/metrics.rs

agentics_domain/models/challenge/
metrics.rs
