agentics_domain/models/challenge/
metrics.rs

1use serde::{Deserialize, Serialize};
2
3use super::super::names::MetricName;
4use super::serde_helpers::{
5    required_nullable, required_nullable_non_empty_vec, required_nullable_non_empty_vec_schema,
6    required_nullable_schema, serialize_empty_vec_as_null,
7};
8
9/// Whether a metric is better when it is larger or smaller.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
11#[serde(rename_all = "snake_case")]
12pub enum MetricDirection {
13    Maximize,
14    Minimize,
15}
16
17/// Visibility level for a metric emitted by the evaluator.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
19#[serde(rename_all = "snake_case")]
20pub enum MetricVisibility {
21    /// Visible in validation feedback and official result views.
22    Public,
23    /// Visible only after a ranking-visible official evaluation.
24    Official,
25}
26
27/// One metric that an evaluator may emit in aggregate or per-run result payloads.
28#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
29#[serde(deny_unknown_fields)]
30pub struct MetricDefinitionSpec {
31    pub name: MetricName,
32    pub label: String,
33    #[serde(deserialize_with = "required_nullable")]
34    #[schemars(required, schema_with = "required_nullable_schema::<String>")]
35    pub unit: Option<String>,
36    pub direction: MetricDirection,
37    pub visibility: MetricVisibility,
38    #[serde(deserialize_with = "required_nullable")]
39    #[schemars(required, schema_with = "required_nullable_schema::<String>")]
40    pub metric_description: Option<String>,
41}
42
43/// Ranking configuration for a challenge.
44#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
45#[serde(deny_unknown_fields)]
46pub struct RankingSpec {
47    pub primary_metric_name: MetricName,
48    #[serde(
49        deserialize_with = "required_nullable_non_empty_vec",
50        serialize_with = "serialize_empty_vec_as_null"
51    )]
52    #[schemars(
53        required,
54        schema_with = "required_nullable_non_empty_vec_schema::<MetricName>"
55    )]
56    pub tie_breaker_metric_names: Vec<MetricName>,
57}
58
59/// Metric schema embedded in `spec.json`.
60#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
61#[serde(deny_unknown_fields)]
62pub struct MetricSchemaSpec {
63    pub metrics: Vec<MetricDefinitionSpec>,
64    pub ranking: RankingSpec,
65}
66
67impl MetricSchemaSpec {
68    /// Look up a metric definition by name.
69    pub fn metric(&self, metric_name: &MetricName) -> Option<&MetricDefinitionSpec> {
70        self.metrics
71            .iter()
72            .find(|metric| &metric.name == metric_name)
73    }
74
75    /// Primary ranking metric declared by this challenge.
76    pub fn primary_metric(&self) -> Option<&MetricDefinitionSpec> {
77        self.metric(&self.ranking.primary_metric_name)
78    }
79}
80
81impl Default for MetricSchemaSpec {
82    /// Handles default for this module.
83    fn default() -> Self {
84        Self {
85            metrics: vec![MetricDefinitionSpec {
86                name: MetricName::score(),
87                label: "Score".to_string(),
88                unit: None,
89                direction: MetricDirection::Maximize,
90                visibility: MetricVisibility::Public,
91                metric_description: Some("Challenge-defined compatibility score.".to_string()),
92            }],
93            ranking: RankingSpec {
94                primary_metric_name: MetricName::score(),
95                tie_breaker_metric_names: vec![],
96            },
97        }
98    }
99}
agentics_domain/models/challenge/metrics.rs

agentics_domain/models/challenge/
metrics.rs
