rustev_surface/
adapter.rs

1//! The declarative task adapter, `rustev.task-adapter/1` (spec 006, 3.8):
2//! how evaluation reads labels and decides whether a proposal is correct,
3//! without writing Rust.
4
5use std::collections::{BTreeMap, BTreeSet};
6
7use rustev_contract::Document;
8use rustev_contract::bounded::ParseLimits;
9use rustev_contract::canonical::tagged_digest;
10use rustev_contract::ids::ContentDigest;
11use rustev_contract::judgment::OutValue;
12use rustev_contract::limits::DESCRIPTOR_V1;
13use rustev_eval::config::AdapterRules;
14use rustev_eval::dataset::AdapterRef;
15use rustev_eval::metrics::TaskAdapter;
16use serde::{Deserialize, Serialize};
17
18pub const TASK_ADAPTER: &str = "rustev.task-adapter/1";
19const MAX_RULES: usize = 64;
20const MAX_LABELS: usize = 1024;
21const MAX_TEXT: usize = 256;
22
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24#[serde(deny_unknown_fields)]
25pub struct TaskAdapterDoc {
26    pub schema: String,
27    pub name: String,
28    pub version: String,
29    pub labels: Labels,
30    pub rules: Vec<Rule>,
31}
32
33/// A label is well formed when it equals one of `values` or starts with one
34/// of `prefixes`.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
36#[serde(deny_unknown_fields)]
37pub struct Labels {
38    pub values: Vec<String>,
39    pub prefixes: Vec<String>,
40}
41
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
43#[serde(untagged)]
44pub enum Rule {
45    /// The proposal is correct for exactly this label.
46    Label(LabelRule),
47    /// The proposal is correct when a selected parameter value, mapped,
48    /// equals the label.
49    Param(ParamRule),
50}
51
52#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
53#[serde(deny_unknown_fields)]
54pub struct LabelRule {
55    pub action: String,
56    pub label: String,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
60#[serde(deny_unknown_fields)]
61pub struct ParamRule {
62    pub action: String,
63    pub param: String,
64    pub select: Select,
65    pub map: BTreeMap<String, String>,
66    pub otherwise: Option<String>,
67}
68
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
70#[serde(rename_all = "snake_case")]
71pub enum Select {
72    Enum,
73    Text,
74    Integer,
75    FirstRanked,
76}
77
78impl Document for TaskAdapterDoc {
79    const SCHEMA: &'static str = TASK_ADAPTER;
80    const LIMITS: ParseLimits = DESCRIPTOR_V1;
81    fn schema(&self) -> &str {
82        &self.schema
83    }
84}
85
86impl Rule {
87    fn action(&self) -> &str {
88        match self {
89            Rule::Label(r) => &r.action,
90            Rule::Param(r) => &r.action,
91        }
92    }
93}
94
95fn text(what: &str, s: &str, max: usize) -> Result<(), String> {
96    if s.is_empty() || s.len() > max {
97        return Err(format!("{what} is 1 to {max} bytes"));
98    }
99    Ok(())
100}
101
102impl TaskAdapterDoc {
103    /// Validate the adapter (spec 006, 3.8.1).
104    pub fn check(&self) -> Result<(), String> {
105        text("name", &self.name, 64)?;
106        text("version", &self.version, 64)?;
107        let l = &self.labels;
108        if l.values.is_empty() && l.prefixes.is_empty() {
109            return Err("labels need at least one value or prefix".into());
110        }
111        if l.values.len() + l.prefixes.len() > MAX_LABELS {
112            return Err(format!("at most {MAX_LABELS} label values and prefixes"));
113        }
114        for v in l.values.iter().chain(&l.prefixes) {
115            text("a label value or prefix", v, MAX_TEXT)?;
116        }
117        if self.rules.len() > MAX_RULES {
118            return Err(format!("at most {MAX_RULES} rules"));
119        }
120        let mut actions = BTreeSet::new();
121        for r in &self.rules {
122            text("an action", r.action(), MAX_TEXT)?;
123            if !actions.insert(r.action()) {
124                return Err(format!("two rules for action {:?}", r.action()));
125            }
126            match r {
127                Rule::Label(x) => text("a rule label", &x.label, MAX_TEXT)?,
128                Rule::Param(x) => {
129                    text("a parameter name", &x.param, MAX_TEXT)?;
130                    if x.map.len() > MAX_LABELS {
131                        return Err(format!("a map has at most {MAX_LABELS} entries"));
132                    }
133                    for (k, v) in &x.map {
134                        text("a map key", k, MAX_TEXT)?;
135                        text("a map value", v, MAX_TEXT)?;
136                    }
137                    if let Some(o) = &x.otherwise {
138                        text("otherwise", o, MAX_TEXT)?;
139                    }
140                }
141            }
142        }
143        Ok(())
144    }
145}
146
147/// The value a parameter rule selects, or none when the parameter is
148/// missing or of another type.
149fn selected(select: Select, v: Option<&OutValue>) -> Option<String> {
150    match (select, v?) {
151        (Select::Enum, OutValue::Enum(s)) | (Select::Text, OutValue::Text(s)) => Some(s.clone()),
152        (Select::Integer, OutValue::Integer(i)) => Some(i.to_string()),
153        (Select::FirstRanked, OutValue::Ranking(r)) => {
154            r.entries.first().map(|e| e.candidate.clone())
155        }
156        _ => None,
157    }
158}
159
160impl TaskAdapter for TaskAdapterDoc {
161    fn adapter(&self) -> AdapterRef {
162        AdapterRef {
163            name: self.name.clone(),
164            version: self.version.clone(),
165        }
166    }
167
168    /// Always bound: the document is the whole definition (spec 014,
169    /// 3.1.4), and the digest is over the canonical bytes `eval` writes as
170    /// `adapter.json`.
171    fn rules(&self) -> AdapterRules {
172        let bytes = self
173            .canonical()
174            .expect("a checked adapter document has a canonical form");
175        AdapterRules::Bound(
176            ContentDigest::parse(&tagged_digest(TASK_ADAPTER, &bytes))
177                .expect("a tagged digest is a content digest"),
178        )
179    }
180
181    fn check_label(&self, label: &str) -> Result<(), String> {
182        let l = &self.labels;
183        if l.values.iter().any(|v| v == label)
184            || l.prefixes.iter().any(|p| label.starts_with(p.as_str()))
185        {
186            Ok(())
187        } else {
188            Err(format!("{label:?} is not a label of adapter {}", self.name))
189        }
190    }
191
192    fn correct(&self, action: &str, params: &BTreeMap<String, OutValue>, label: &str) -> bool {
193        let Some(rule) = self.rules.iter().find(|r| r.action() == action) else {
194            return false;
195        };
196        match rule {
197            Rule::Label(r) => r.label == label,
198            Rule::Param(r) => {
199                let Some(value) = selected(r.select, params.get(&r.param)) else {
200                    return false;
201                };
202                let mapped = if r.map.is_empty() {
203                    Some(value)
204                } else {
205                    r.map.get(&value).cloned().or_else(|| r.otherwise.clone())
206                };
207                mapped.is_some_and(|m| m == label)
208            }
209        }
210    }
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216    use rustev_contract::judgment::{RankEntry, Ranking};
217
218    fn support() -> TaskAdapterDoc {
219        TaskAdapterDoc {
220            schema: TASK_ADAPTER.into(),
221            name: "support-routing".into(),
222            version: "1".into(),
223            labels: Labels {
224                values: ["billing", "integration_defect", "account_access", "other"]
225                    .map(String::from)
226                    .to_vec(),
227                prefixes: vec![],
228            },
229            rules: vec![Rule::Param(ParamRule {
230                action: "route_ticket".into(),
231                param: "queue".into(),
232                select: Select::Enum,
233                map: [
234                    ("billing", "billing"),
235                    ("billing-priority", "billing"),
236                    ("engineering", "integration_defect"),
237                    ("identity", "account_access"),
238                ]
239                .into_iter()
240                .map(|(a, b)| (a.to_string(), b.to_string()))
241                .collect(),
242                otherwise: Some("other".into()),
243            })],
244        }
245    }
246
247    fn params(v: &[(&str, OutValue)]) -> BTreeMap<String, OutValue> {
248        v.iter().map(|(k, x)| (k.to_string(), x.clone())).collect()
249    }
250
251    #[test]
252    fn a_parameter_rule_maps_and_falls_back_to_otherwise() {
253        let a = support();
254        a.check().unwrap();
255        let q = |s: &str| params(&[("queue", OutValue::Enum(s.into()))]);
256        assert!(a.correct("route_ticket", &q("billing-priority"), "billing"));
257        assert!(!a.correct("route_ticket", &q("billing-priority"), "other"));
258        assert!(a.correct("route_ticket", &q("general"), "other"));
259        // A missing parameter, another type or another action is never correct.
260        assert!(!a.correct("route_ticket", &params(&[]), "other"));
261        let text = params(&[("queue", OutValue::Text("billing".into()))]);
262        assert!(!a.correct("route_ticket", &text, "billing"));
263        assert!(!a.correct("escalate", &q("billing"), "billing"));
264        // Without `otherwise`, an unmapped value is never correct.
265        let mut b = support();
266        if let Rule::Param(r) = &mut b.rules[0] {
267            r.otherwise = None;
268        }
269        assert!(!b.correct("route_ticket", &q("general"), "other"));
270    }
271
272    #[test]
273    fn label_rules_first_ranked_and_label_shapes() {
274        let a = TaskAdapterDoc {
275            schema: TASK_ADAPTER.into(),
276            name: "lodging".into(),
277            version: "1".into(),
278            labels: Labels {
279                values: vec!["none".into()],
280                prefixes: vec!["c-".into()],
281            },
282            rules: vec![
283                Rule::Param(ParamRule {
284                    action: "present_ranked_lodging".into(),
285                    param: "ranking".into(),
286                    select: Select::FirstRanked,
287                    map: BTreeMap::new(),
288                    otherwise: None,
289                }),
290                Rule::Label(LabelRule {
291                    action: "no_eligible_lodging".into(),
292                    label: "none".into(),
293                }),
294            ],
295        };
296        a.check().unwrap();
297        let entry = |c: &str, p| RankEntry {
298            candidate: c.into(),
299            position: p,
300            score: 1.0,
301            components: BTreeMap::new(),
302        };
303        let ranking = |e: Vec<RankEntry>| {
304            params(&[(
305                "ranking",
306                OutValue::Ranking(Ranking {
307                    entries: e,
308                    excluded: vec![],
309                }),
310            )])
311        };
312        let r = ranking(vec![entry("c-1", 1), entry("c-2", 2)]);
313        assert!(a.correct("present_ranked_lodging", &r, "c-1"));
314        assert!(!a.correct("present_ranked_lodging", &r, "c-2"));
315        assert!(!a.correct("present_ranked_lodging", &ranking(vec![]), "c-1"));
316        assert!(a.correct("no_eligible_lodging", &params(&[]), "none"));
317        assert!(!a.correct("no_eligible_lodging", &params(&[]), "c-1"));
318        assert!(a.check_label("c-9").is_ok() && a.check_label("none").is_ok());
319        assert!(a.check_label("x-1").is_err());
320    }
321
322    #[test]
323    fn invalid_adapters_are_refused() {
324        let mut a = support();
325        a.rules.push(a.rules[0].clone());
326        assert!(a.check().unwrap_err().contains("two rules"));
327        let mut a = support();
328        a.labels.values.clear();
329        assert!(a.check().is_err());
330        let mut a = support();
331        a.name = "x".repeat(65);
332        assert!(a.check().is_err());
333        // Unknown fields are refused by the parser.
334        let bytes = br#"{"schema":"rustev.task-adapter/1","name":"n","version":"1","labels":{"values":["a"],"prefixes":[]},"rules":[{"action":"x","label":"a","extra":1}]}"#;
335        assert!(TaskAdapterDoc::parse(bytes).is_err());
336    }
337}
rustev_surface/adapter.rs

rustev_surface/
adapter.rs
