rustev_eval/
gold.rs

1//! Gold records and gold values (spec 041, 3.3).
2//!
3//! A gold record is a host's label for one case: graded relevance of
4//! items, item attributes, requested attributes, expected extraction
5//! fields and human rubric levels. It is not verified truth, and a case
6//! without the member a metric needs is excluded from that metric
7//! (`no_gold`), never read as failing.
8//!
9//! A proposal value matches a gold value by kind only (3.3.3); any other
10//! pairing is a type mismatch, which is neither a match nor a mismatch.
11
12use std::collections::BTreeMap;
13
14use rustev_contract::judgment::OutValue;
15use serde::{Deserialize, Serialize};
16
17/// The highest relevance grade (3.3.1).
18pub const MAX_GRADE: u8 = 10;
19/// The most entries in one gold map (3.3.2).
20pub const MAX_ENTRIES: usize = 1000;
21/// The longest hierarchical path (3.3.2).
22pub const MAX_PATH: usize = 16;
23/// The longest item id, attribute, field or scale name, in bytes (3.3.2).
24pub const MAX_NAME: usize = 256;
25/// The gold value of an extraction field that is expected to be absent
26/// (3.3.1).
27pub const ABSENT: &str = "absent";
28
29/// A case's gold record (3.3.1). Every member is optional.
30#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
31#[serde(deny_unknown_fields)]
32pub struct Gold {
33    /// Item id to grade 0 to 10; an item absent from the map is unjudged.
34    #[serde(default, skip_serializing_if = "Option::is_none")]
35    pub relevance: Option<BTreeMap<String, u8>>,
36    /// Item id to attribute name to gold value.
37    #[serde(default, skip_serializing_if = "Option::is_none")]
38    pub items: Option<BTreeMap<String, BTreeMap<String, GoldValue>>>,
39    /// Request attribute name to gold value.
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub requested: Option<BTreeMap<String, GoldValue>>,
42    /// Extraction field name to gold value, or the string `absent`.
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub fields: Option<BTreeMap<String, GoldValue>>,
45    /// Human scale name to level name.
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub rubric: Option<BTreeMap<String, String>>,
48}
49
50/// A gold value (3.3.2): a string, an integer, a boolean, or a
51/// hierarchical path of 1 to 16 strings, most general first.
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
53#[serde(untagged)]
54pub enum GoldValue {
55    Bool(bool),
56    Integer(i64),
57    Text(String),
58    Path(Vec<String>),
59}
60
61/// What a field's gold says (3.3.1).
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum FieldGold<'a> {
64    Value(&'a GoldValue),
65    Absent,
66}
67
68/// How a proposal value compares with a gold value (3.3.3).
69#[derive(Debug, Clone, Copy, PartialEq, Eq)]
70pub enum Match {
71    Equal,
72    Unequal,
73    /// The kinds do not pair: excluded and counted, never a match or a
74    /// mismatch.
75    TypeMismatch,
76}
77
78impl Match {
79    fn of(equal: bool) -> Match {
80        if equal { Match::Equal } else { Match::Unequal }
81    }
82}
83
84/// Compares a proposal value with a gold value by kind (3.3.3).
85pub fn match_value(value: &OutValue, gold: &GoldValue) -> Match {
86    match (value, gold) {
87        (OutValue::Text(v) | OutValue::Enum(v), GoldValue::Text(g)) => Match::of(v == g),
88        (OutValue::Integer(v), GoldValue::Integer(g)) => Match::of(v == g),
89        (OutValue::Bool(v), GoldValue::Bool(g)) => Match::of(v == g),
90        (OutValue::Decimal(v), GoldValue::Text(g)) => Match::of(&v.to_canonical_string() == g),
91        (OutValue::List(v), GoldValue::Path(g)) => {
92            let mut texts = Vec::with_capacity(v.len());
93            for e in v {
94                match e {
95                    OutValue::Text(t) => texts.push(t),
96                    _ => return Match::TypeMismatch,
97                }
98            }
99            Match::of(texts.len() == g.len() && texts.iter().zip(g).all(|(a, b)| *a == b))
100        }
101        _ => Match::TypeMismatch,
102    }
103}
104
105fn name_ok(s: &str) -> bool {
106    !s.is_empty() && s.len() <= MAX_NAME
107}
108
109fn map_ok<V>(what: &str, m: &BTreeMap<String, V>) -> Result<(), String> {
110    if m.len() > MAX_ENTRIES {
111        return Err(format!("gold {what} holds more than {MAX_ENTRIES} entries"));
112    }
113    if !m.keys().all(|k| name_ok(k)) {
114        return Err(format!("a gold {what} name is 1 to {MAX_NAME} bytes"));
115    }
116    Ok(())
117}
118
119impl GoldValue {
120    /// The path bound of 3.3.2.
121    pub fn check(&self) -> Result<(), String> {
122        match self {
123            GoldValue::Path(p) if p.is_empty() || p.len() > MAX_PATH => {
124                Err(format!("a gold path holds 1 to {MAX_PATH} strings"))
125            }
126            _ => Ok(()),
127        }
128    }
129}
130
131impl Gold {
132    /// The structural rules of 3.3.2; the caller names the case.
133    pub fn check(&self) -> Result<(), String> {
134        if let Some(r) = &self.relevance {
135            map_ok("relevance", r)?;
136            if r.values().any(|g| *g > MAX_GRADE) {
137                return Err(format!("a relevance grade is 0 to {MAX_GRADE}"));
138            }
139        }
140        if let Some(items) = &self.items {
141            map_ok("items", items)?;
142            for attrs in items.values() {
143                map_ok("item attribute", attrs)?;
144                attrs.values().try_for_each(GoldValue::check)?;
145            }
146        }
147        for (what, m) in [("requested", &self.requested), ("fields", &self.fields)] {
148            if let Some(m) = m {
149                map_ok(what, m)?;
150                m.values().try_for_each(GoldValue::check)?;
151            }
152        }
153        if let Some(r) = &self.rubric {
154            // 3.3.2 bounds scale names only. A level is checked against its
155            // declared human scale at evaluation, whose levels are bounded
156            // names, and a level not on the scale refuses the manifest
157            // (3.6.4), so an unbounded level never reaches a metric.
158            map_ok("rubric", r)?;
159        }
160        Ok(())
161    }
162
163    /// A field's gold, when the record states one; the string `absent`
164    /// means the field is expected to be absent.
165    pub fn field(&self, name: &str) -> Option<FieldGold<'_>> {
166        match self.fields.as_ref()?.get(name)? {
167            GoldValue::Text(t) if t == ABSENT => Some(FieldGold::Absent),
168            v => Some(FieldGold::Value(v)),
169        }
170    }
171}
rustev_eval/gold.rs

rustev_eval/
gold.rs
