rustev_eval/
measures.rs

1//! Adapter measures (spec 041, 3.4): the ranking, constraint and field
2//! declarations of `rustev.task-adapter/2`.
3//!
4//! Measures are task semantics, so they belong to the adapter. Their digest
5//! is bound into `rustev.evaluator-config/3` and computed here from the
6//! declarations a [`MeasuredAdapter`] supplies, so a Rust adapter cannot
7//! state one digest and apply other declarations (3.4.1).
8
9use std::collections::BTreeSet;
10
11use rustev_contract::canonical::{CanonicalError, canonical_bytes, tagged_digest};
12use rustev_contract::ids::ContentDigest;
13use serde::{Deserialize, Serialize};
14
15use crate::gold::{GoldValue, MAX_NAME, MAX_PATH};
16use crate::metrics::TaskAdapter;
17use crate::ranking_metrics::{Gain, MAX_K, RankingMetric};
18
19/// Version 2 of the task adapter: version 1 plus `measures` (3.1.2).
20pub const TASK_ADAPTER_V2: &str = "rustev.task-adapter/2";
21/// The most declarations in one list (3.4.1).
22pub const MAX_DECLARATIONS: usize = 16;
23/// The most positions a ranking reports a backfill rate for (3.4.2).
24pub const MAX_BACKFILL_POSITIONS: u64 = 100;
25
26/// An adapter that declares measures; used only under version 3 (3.4.1).
27pub trait MeasuredAdapter: TaskAdapter {
28    fn measures(&self) -> Measures;
29}
30
31#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(deny_unknown_fields)]
33pub struct Measures {
34    pub rankings: Vec<RankingDecl>,
35    pub constraints: Vec<ConstraintDecl>,
36    pub fields: Vec<FieldDecl>,
37}
38
39/// A ranking declaration (3.4.2).
40#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
41#[serde(deny_unknown_fields)]
42pub struct RankingDecl {
43    pub name: String,
44    /// The proposal parameter holding the ranked list.
45    pub param: String,
46    /// 1 to 10: the lowest grade that counts as relevant.
47    pub relevant_at_least: u8,
48    pub gain: Gain,
49    pub metrics: Vec<RankingMetric>,
50    /// 0 to 100: `backfill@p<p>` for p from 1 to this.
51    pub backfill_positions: u64,
52}
53
54/// A constraint declaration (3.4.5).
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct ConstraintDecl {
58    pub name: String,
59    /// A declared ranking whose top k is checked.
60    pub ranking: String,
61    pub k: u64,
62    /// An item attribute in gold `items`.
63    pub attribute: String,
64    pub test: ConstraintTest,
65    pub against: Against,
66    pub aggregate: ConstraintAggregate,
67}
68
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "snake_case")]
71pub enum ConstraintTest {
72    Equals,
73    /// The reference is a path whose elements are the allowed strings.
74    MemberOf,
75    /// Both values are paths and their first `level` elements, 1 to 16,
76    /// are equal.
77    PathPrefix(u8),
78}
79
80/// What an item attribute is checked against (3.4.5).
81#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
82#[serde(rename_all = "snake_case")]
83pub enum Against {
84    /// A gold `requested` value.
85    Requested(String),
86    /// A proposal parameter, matched by 3.3.3 kinds.
87    Param(String),
88    Value(GoldValue),
89}
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
92#[serde(rename_all = "snake_case")]
93pub enum ConstraintAggregate {
94    ShareAtK,
95    AllAtK,
96}
97
98/// A field declaration (3.4.7): an extract step's output field.
99#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
100#[serde(deny_unknown_fields)]
101pub struct FieldDecl {
102    pub name: String,
103    pub step: String,
104    pub field: String,
105}
106
107/// 1 to 64 bytes of `[A-Za-z0-9._-]` (3.4.1).
108pub fn declared_name(s: &str) -> bool {
109    (1..=64).contains(&s.len())
110        && s.bytes()
111            .all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'_' || b == b'-')
112}
113
114fn reference_name(s: &str) -> bool {
115    !s.is_empty() && s.len() <= MAX_NAME
116}
117
118fn unique<'a>(what: &str, names: impl Iterator<Item = &'a str>) -> Result<(), String> {
119    let mut seen = BTreeSet::new();
120    for n in names {
121        if !declared_name(n) {
122            return Err(format!("a {what} name is 1 to 64 bytes of [A-Za-z0-9._-]"));
123        }
124        if !seen.insert(n) {
125            return Err(format!("{what} names are unique"));
126        }
127    }
128    Ok(())
129}
130
131impl Measures {
132    /// The structural rules of 3.4.
133    pub fn check(&self) -> Result<(), String> {
134        for (what, n) in [
135            ("rankings", self.rankings.len()),
136            ("constraints", self.constraints.len()),
137            ("fields", self.fields.len()),
138        ] {
139            if n > MAX_DECLARATIONS {
140                return Err(format!("at most {MAX_DECLARATIONS} {what}"));
141            }
142        }
143        unique("ranking", self.rankings.iter().map(|r| r.name.as_str()))?;
144        unique(
145            "constraint",
146            self.constraints.iter().map(|c| c.name.as_str()),
147        )?;
148        unique("field", self.fields.iter().map(|f| f.name.as_str()))?;
149        for r in &self.rankings {
150            if !reference_name(&r.param) {
151                return Err("a ranking parameter is 1 to 256 bytes".into());
152            }
153            if !(1..=10).contains(&r.relevant_at_least) {
154                return Err("relevant_at_least is 1 to 10".into());
155            }
156            let specs: BTreeSet<_> = r.metrics.iter().collect();
157            if specs.len() != r.metrics.len() {
158                return Err("a ranking's metrics are unique".into());
159            }
160            if r.backfill_positions > MAX_BACKFILL_POSITIONS {
161                return Err("backfill_positions is 0 to 100".into());
162            }
163        }
164        for c in &self.constraints {
165            if !self.rankings.iter().any(|r| r.name == c.ranking) {
166                return Err("a constraint names a declared ranking".into());
167            }
168            if !(1..=MAX_K).contains(&c.k) {
169                return Err("a constraint's k is 1 to 100".into());
170            }
171            if !reference_name(&c.attribute) {
172                return Err("a constraint attribute is 1 to 256 bytes".into());
173            }
174            if let ConstraintTest::PathPrefix(level) = c.test
175                && !(1..=MAX_PATH).contains(&usize::from(level))
176            {
177                return Err("a path_prefix level is 1 to 16".into());
178            }
179            match &c.against {
180                Against::Requested(n) | Against::Param(n) if !reference_name(n) => {
181                    return Err("a constraint reference is 1 to 256 bytes".into());
182                }
183                Against::Value(v) => v.check()?,
184                _ => {}
185            }
186        }
187        for f in &self.fields {
188            if !reference_name(&f.step) || !reference_name(&f.field) {
189                return Err("a field declaration's step and field are 1 to 256 bytes".into());
190            }
191        }
192        Ok(())
193    }
194
195    /// The configuration's adapter `measures` member (3.1.3): the digest of
196    /// the declarations' canonical bytes, tagged with the adapter's schema.
197    pub fn digest(&self) -> Result<ContentDigest, CanonicalError> {
198        let digest = tagged_digest(TASK_ADAPTER_V2, &canonical_bytes(self)?);
199        Ok(ContentDigest::parse(&digest).expect("a tagged digest is a content digest"))
200    }
201}
rustev_eval/measures.rs

rustev_eval/
measures.rs
