rustev_eval/
host_run.rs

1//! Host run records and their case measures, `rustev.host-run/1` and
2//! `rustev.host-cases/1` (spec 061, 3.1 to 3.3).
3//!
4//! A host whose runs execute in its own runtime supplies one record per
5//! run, naming its views and scores by record digest. Rustev checks the
6//! record's fields, binds the views and scores to it, and computes each
7//! case's measures: the score metrics of spec 041 over the answered cases
8//! and the `host/` metrics over every case. Every figure here is computed
9//! from host-asserted observations: Rustev did not observe the run, and no
10//! host document carries a Rustev plan or dataset identity (I-2).
11
12use std::collections::{BTreeMap, BTreeSet};
13use std::fmt;
14
15use rustev_contract::decimal::Decimal;
16use rustev_contract::eval_report::{MetricValue, Split};
17use rustev_contract::ids::{ContentDigest, SnapshotId};
18use rustev_contract::judgment::Derivation;
19use rustev_contract::limits::REPLAY_V1;
20use rustev_contract::scope::Handle;
21use rustev_contract::{Document, Identified};
22use serde::{Deserialize, Serialize};
23
24use crate::config::EvaluatorConfig;
25use crate::dataset::DatasetCase;
26use crate::eval_cases::CaseMeasure;
27use crate::measured::{self, Scope};
28use crate::scores::{ScoreOutcome, ScoreViews, Scores, reason_ok, score_ok};
29use crate::scoring::Declarations;
30
31pub const HOST_RUN: &str = "rustev.host-run/1";
32pub const HOST_CASES: &str = "rustev.host-cases/1";
33/// At most one scores document per scorer, and at most 16 scorers (3.1.5).
34pub const MAX_SCORES: usize = 16;
35/// The largest latency a record states, in microseconds (3.1.4).
36pub const MAX_LATENCY_US: u64 = 1 << 53;
37
38/// The `host/` metrics (3.3.3), in report order.
39pub mod metric {
40    pub const ANSWERED_RATE: &str = "host/answered_rate";
41    pub const ABSTAINED_RATE: &str = "host/abstained_rate";
42    pub const FAILED_RATE: &str = "host/failed_rate";
43    pub const SCORED_RATE: &str = "host/scored_rate";
44    pub const LATENCY_P50: &str = "host/latency_ms/p0.5";
45    pub const LATENCY_P95: &str = "host/latency_ms/p0.95";
46    pub const COST_PER_1000: &str = "host/cost_per_1000";
47    pub const ALL: [&str; 7] = [
48        ANSWERED_RATE,
49        ABSTAINED_RATE,
50        FAILED_RATE,
51        SCORED_RATE,
52        LATENCY_P50,
53        LATENCY_P95,
54        COST_PER_1000,
55    ];
56}
57
58/// Why `host/cost_per_1000` is unknown when the record declares no unit.
59pub const COST_NOT_DECLARED: &str = "cost-not-declared";
60
61/// Every host document states that its observations are the host's
62/// (3.5.3).
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
64#[serde(rename_all = "snake_case")]
65pub enum Assertion {
66    Host,
67}
68
69/// One host run (3.1.1). Its record digest is the run identity Rustev
70/// binds.
71#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct HostRun {
74    pub schema: String,
75    pub host: HostIdentity,
76    pub dataset: HostDataset,
77    pub split: Split,
78    /// The record digest of the run's `rustev.score-views/1`.
79    pub views: ContentDigest,
80    /// The record digests of its `rustev.scores/1` documents.
81    pub scores: Vec<ContentDigest>,
82    pub cost_unit: Option<String>,
83    /// Supplied by the host, never read from a clock.
84    pub recorded_at_ms: i64,
85    /// In the host's dataset order.
86    pub cases: Vec<HostCase>,
87}
88
89crate::doc::eval_document!(HostRun, HOST_RUN, REPLAY_V1);
90
91#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
92#[serde(deny_unknown_fields)]
93pub struct HostIdentity {
94    pub system: Handle,
95    pub flow: Handle,
96    pub flow_version: Handle,
97    pub run: Handle,
98}
99
100/// The host's dataset as it asserts it; Rustev reads no dataset bytes.
101#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
102#[serde(deny_unknown_fields)]
103pub struct HostDataset {
104    pub id: Handle,
105    pub digest: ContentDigest,
106    /// The dataset's case count for the split.
107    pub cases: u64,
108}
109
110#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
111#[serde(deny_unknown_fields)]
112pub struct HostCase {
113    pub case: String,
114    pub subgroups: Vec<String>,
115    pub outcome: HostOutcome,
116    pub latency_us: HostLatency,
117    pub cost: Option<HostCost>,
118}
119
120#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
121#[serde(rename_all = "snake_case", deny_unknown_fields)]
122pub enum HostOutcome {
123    Answered,
124    Abstained,
125    Failed(String),
126}
127
128/// `{ "unknown": reason }`: a value the host did not observe (I-3).
129#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
130#[serde(deny_unknown_fields)]
131pub struct Unobserved {
132    pub unknown: String,
133}
134
135#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
136#[serde(untagged)]
137pub enum HostLatency {
138    Us(u64),
139    Unknown(Unobserved),
140}
141
142#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
143#[serde(untagged)]
144pub enum HostCost {
145    Value(Decimal),
146    Unknown(Unobserved),
147}
148
149/// A refusal of spec 061 (`invalid_input`, 3.9), naming the run when it is
150/// one run's, and its code.
151#[derive(Debug, Clone, PartialEq, Eq)]
152pub struct HostError {
153    pub run: Option<usize>,
154    pub code: &'static str,
155    pub detail: String,
156}
157
158impl HostError {
159    pub fn new(code: &'static str, detail: impl Into<String>) -> HostError {
160        HostError {
161            run: None,
162            code,
163            detail: detail.into(),
164        }
165    }
166
167    pub fn at(mut self, run: usize) -> HostError {
168        self.run = Some(run);
169        self
170    }
171}
172
173impl fmt::Display for HostError {
174    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
175        if let Some(r) = self.run {
176            write!(f, "run {r}: ")?;
177        }
178        write!(f, "{}: {}", self.code, self.detail)
179    }
180}
181
182impl std::error::Error for HostError {}
183
184/// Refusal codes (3.2, 3.4, 3.6).
185pub mod code {
186    pub const HOST_RUN: &str = "host-run";
187    pub const HOST_BINDING: &str = "host-binding";
188    pub const CONFIG: &str = "config";
189    pub const MIXED_RUNS: &str = "mixed-runs";
190    pub const REPEAT_CONFIG: &str = "repeat-config";
191    pub const RECONCILE: &str = "reconcile";
192    pub const MIXED_PROVENANCE: &str = "mixed-provenance";
193}
194
195fn run_err(detail: impl Into<String>) -> HostError {
196    HostError::new(code::HOST_RUN, detail)
197}
198
199fn binding(detail: impl Into<String>) -> HostError {
200    HostError::new(code::HOST_BINDING, detail)
201}
202
203/// Spec 006 3.9.1's case id rule: 1 to 128 bytes of `[A-Za-z0-9._-]`,
204/// starting alphanumeric.
205fn case_id_ok(id: &str) -> bool {
206    !id.is_empty()
207        && id.len() <= 128
208        && id.as_bytes()[0].is_ascii_alphanumeric()
209        && id
210            .bytes()
211            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
212}
213
214/// 1 to 64 bytes of `[a-z0-9_.]` (3.1.2).
215fn unit_ok(u: &str) -> bool {
216    (1..=64).contains(&u.len())
217        && u.bytes()
218            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_' || b == b'.')
219}
220
221impl HostRun {
222    /// The field rules of 3.1 the parser does not state (`host-run`).
223    pub fn check(&self) -> Result<(), HostError> {
224        if self.schema != HOST_RUN {
225            return Err(run_err("wrong schema"));
226        }
227        if self.cases.len() as u64 != self.dataset.cases {
228            return Err(run_err(format!(
229                "{} cases listed, the dataset states {}",
230                self.cases.len(),
231                self.dataset.cases
232            )));
233        }
234        if self.scores.len() > MAX_SCORES {
235            return Err(run_err("at most 16 scores documents"));
236        }
237        if self.scores.iter().collect::<BTreeSet<_>>().len() != self.scores.len() {
238            return Err(run_err("a scores digest is named twice"));
239        }
240        if let Some(u) = &self.cost_unit
241            && !unit_ok(u)
242        {
243            return Err(run_err("a cost unit is 1 to 64 bytes of [a-z0-9_.]"));
244        }
245        let mut ids = BTreeSet::new();
246        for c in &self.cases {
247            let at = |d: &str| run_err(format!("case {:?}: {d}", c.case));
248            if !case_id_ok(&c.case) || !ids.insert(c.case.to_ascii_lowercase()) {
249                return Err(at("not a case id, or not unique ignoring ASCII case"));
250            }
251            let groups: BTreeSet<&String> = c.subgroups.iter().collect();
252            if groups.len() != c.subgroups.len()
253                || c.subgroups.iter().any(|g| g.is_empty() || g.len() > 256)
254            {
255                return Err(at("subgroups are unique names of 1 to 256 bytes"));
256            }
257            if let HostOutcome::Failed(r) = &c.outcome
258                && !reason_ok(r)
259            {
260                return Err(at("a reason is 1 to 64 bytes of [a-z0-9_]"));
261            }
262            match &c.latency_us {
263                HostLatency::Us(us) if *us > MAX_LATENCY_US => {
264                    return Err(at("latency above 2^53 microseconds"));
265                }
266                HostLatency::Unknown(u) if !reason_ok(&u.unknown) => {
267                    return Err(at("a reason is 1 to 64 bytes of [a-z0-9_]"));
268                }
269                _ => {}
270            }
271            match (&self.cost_unit, &c.cost) {
272                (None, None) => {}
273                (None, Some(_)) => return Err(at("a cost without a cost unit")),
274                // Missing cost under a declared unit is refused, never zero.
275                (Some(_), None) => return Err(at("no cost under a declared cost unit")),
276                (Some(_), Some(HostCost::Value(d))) if d.is_negative() => {
277                    return Err(at("a negative cost"));
278                }
279                (Some(_), Some(HostCost::Unknown(u))) if !reason_ok(&u.unknown) => {
280                    return Err(at("a reason is 1 to 64 bytes of [a-z0-9_]"));
281                }
282                _ => {}
283            }
284        }
285        Ok(())
286    }
287
288    /// The record's `answered` cases, in record order.
289    pub fn answered(&self) -> impl Iterator<Item = &HostCase> {
290        self.cases
291            .iter()
292            .filter(|c| c.outcome == HostOutcome::Answered)
293    }
294}
295
296/// One run's case measures (3.3.1): spec 029's per-case document, bound to
297/// the host run record instead of a run report.
298#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
299#[serde(deny_unknown_fields)]
300pub struct HostCases {
301    pub schema: String,
302    pub assertion: Assertion,
303    /// The record digest of the run's `rustev.host-run/1`.
304    pub host_run: ContentDigest,
305    pub cases: Vec<HostCaseMeasures>,
306}
307
308crate::doc::eval_document!(HostCases, HOST_CASES, REPLAY_V1);
309
310#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
311#[serde(deny_unknown_fields)]
312pub struct HostCaseMeasures {
313    pub case: String,
314    pub outcome: HostOutcome,
315    /// Per metric, the case's contribution; a metric whose denominator the
316    /// case is outside has no entry.
317    pub measures: BTreeMap<String, CaseMeasure>,
318}
319
320/// One bound run: its record, views and scores documents, with the
321/// evaluator configuration that declares the scorers.
322#[derive(Clone, Copy)]
323pub struct HostRunInputs<'a> {
324    pub record: &'a HostRun,
325    /// None when the run supplied no views document.
326    pub views: Option<&'a ScoreViews>,
327    pub scores: &'a [Scores],
328}
329
330/// A run's measures and its per-run metric values, in report order.
331#[derive(Debug, Clone, PartialEq)]
332pub struct HostMeasured {
333    pub cases: HostCases,
334    /// Every metric of the run, in report order.
335    pub metrics: Vec<HostMetric>,
336}
337
338#[derive(Debug, Clone, PartialEq)]
339pub struct HostMetric {
340    pub name: String,
341    pub derivation: Derivation,
342    /// Whether an opaque scorer produces it (spec 041, 3.12).
343    pub opaque: bool,
344    /// A score metric's run value, as spec 041's function reports it: the
345    /// only figure of a metric with no per-case measure (`qwk`); none for
346    /// a `host/` metric, whose run values come from its measures.
347    pub value: Option<MetricValue>,
348}
349
350/// The configuration's version 3 declarations, or `config`.
351pub fn declarations(config: &EvaluatorConfig) -> Result<Declarations<'_>, HostError> {
352    config
353        .check()
354        .map_err(|e| HostError::new(code::CONFIG, e.to_string()))?;
355    config.declarations().ok_or_else(|| {
356        HostError::new(
357            code::CONFIG,
358            "a host set needs a rustev.evaluator-config/3 configuration",
359        )
360    })
361}
362
363/// The binding checks of 3.2.2, before any statistic: the record's fields,
364/// the views and scores digests it names, the views' configuration, split,
365/// cases and subgroups, each entry's subject, and spec 041 3.7.5's scores
366/// checks.
367pub fn bind(config: &EvaluatorConfig, run: &HostRunInputs<'_>) -> Result<(), HostError> {
368    let declared = declarations(config)?;
369    let record = run.record;
370    record.check()?;
371    let views = run
372        .views
373        .ok_or_else(|| binding("the record names views, and no views document was supplied"))?;
374    let digest = views
375        .record_digest()
376        .map_err(|e| binding(format!("the views document has no record digest: {e}")))?;
377    if digest != record.views {
378        return Err(binding(
379            "the views document is not the one the record names",
380        ));
381    }
382    let mut named: BTreeSet<&ContentDigest> = record.scores.iter().collect();
383    for s in run.scores {
384        let d = s
385            .record_digest()
386            .map_err(|e| binding(format!("a scores document has no record digest: {e}")))?;
387        if !named.remove(&d) {
388            return Err(binding(format!(
389                "the scores document of scorer {:?} is not one the record names",
390                s.scorer.id
391            )));
392        }
393    }
394    if let Some(d) = named.first() {
395        return Err(binding(format!(
396            "the record names scores {d}, and no such document was supplied"
397        )));
398    }
399    let config_id = config
400        .id()
401        .map_err(|e| HostError::new(code::CONFIG, e.to_string()))?;
402    if views.config != config_id {
403        return Err(binding("the views name another evaluator configuration"));
404    }
405    if views.split != record.split {
406        return Err(binding("the views name another split"));
407    }
408    let answered: Vec<&HostCase> = record.answered().collect();
409    if views.cases.len() != answered.len() {
410        return Err(binding(format!(
411            "the views list {} cases, the record {} answered",
412            views.cases.len(),
413            answered.len()
414        )));
415    }
416    for (v, c) in views.cases.iter().zip(&answered) {
417        if v.case != c.case || v.case_view.id != c.case {
418            return Err(binding(format!(
419                "views case {:?} is not the record's answered case {:?}",
420                v.case, c.case
421            )));
422        }
423        if v.case_view.subgroups != c.subgroups {
424            return Err(binding(format!(
425                "case {:?}: the views' subgroups differ from the record's",
426                c.case
427            )));
428        }
429        if v.proposal_view.subject().ok().as_ref() != Some(&v.subject) {
430            return Err(binding(format!(
431                "case {:?}: the subject is not the proposal view's digest",
432                c.case
433            )));
434        }
435    }
436    crate::report::scores_refusal(&declared, run.scores, views, &digest)
437        .map_err(|c| HostError::new(c, "spec 041, 3.7.5"))
438}
439
440fn count(hit: bool) -> CaseMeasure {
441    CaseMeasure::Count {
442        num: u64::from(hit),
443        den: 1,
444    }
445}
446
447/// One `answered` case as spec 041's metric functions read it: its id,
448/// label, gold and subgroups from the views. The source and snapshot are
449/// placeholders those functions never read; the case is never written.
450fn view_case(
451    entry: &crate::scores::ViewEntry,
452    split: Split,
453    digest: &ContentDigest,
454) -> DatasetCase {
455    DatasetCase {
456        id: entry.case.clone(),
457        source: "host".into(),
458        snapshot: SnapshotId::parse(digest.as_str()).expect("a content digest is a digest"),
459        split,
460        label: entry.case_view.label.clone(),
461        subgroups: entry.case_view.subgroups.clone(),
462        gold: entry.case_view.gold.clone(),
463    }
464}
465
466/// Measure one bound run (3.3): the `host/` metrics over every case, then
467/// the score metrics of spec 041 over the `answered` cases, deterministic
468/// ones first, by the functions spec 041's evaluation uses. Metrics spec
469/// 004 or 041 define from Rustev decision records are not produced.
470pub fn measure(
471    config: &EvaluatorConfig,
472    run: &HostRunInputs<'_>,
473) -> Result<HostMeasured, HostError> {
474    bind(config, run)?;
475    let declared = declarations(config)?;
476    let record = run.record;
477    let views = run.views.expect("bound");
478    let names = declared.metric_names();
479
480    let cases: Vec<DatasetCase> = views
481        .cases
482        .iter()
483        .map(|e| view_case(e, record.split, &record.dataset.digest))
484        .collect();
485    let mut next = 0;
486    let scope: Vec<Scope<'_>> = record
487        .cases
488        .iter()
489        .map(|c| match &c.outcome {
490            HostOutcome::Answered => {
491                let i = next;
492                next += 1;
493                Scope::Proposal {
494                    case: &cases[i],
495                    params: &views.cases[i].proposal_view.params,
496                }
497            }
498            HostOutcome::Abstained => Scope::Out("abstained".into()),
499            HostOutcome::Failed(_) => Scope::Out("failed".into()),
500        })
501        .collect();
502    let (rubric, model_rubric) = measured::rubric_metrics(&declared, run.scores, &scope);
503    let (agreement, model_agreement) = measured::agreement_metrics(&declared, run.scores, &scope);
504    let scored: Vec<bool> = {
505        let mut answered = views.cases.iter();
506        record
507            .cases
508            .iter()
509            .map(|c| {
510                c.outcome == HostOutcome::Answered && {
511                    let e = answered.next().expect("bound");
512                    declared.scorers.iter().all(|s| {
513                        run.scores
514                            .iter()
515                            .find(|d| d.scorer.id == s.id)
516                            .and_then(|d| d.entries.iter().find(|x| x.case == e.case))
517                            .is_some_and(|x| {
518                                matches!(x.outcome, ScoreOutcome::Score(v) if score_ok(&s.output, v))
519                            })
520                    })
521                }
522            })
523            .collect()
524    };
525
526    let mut out: Vec<HostCaseMeasures> = record
527        .cases
528        .iter()
529        .zip(&scored)
530        .map(|(c, scored)| {
531            let mut m = BTreeMap::new();
532            let answered = c.outcome == HostOutcome::Answered;
533            let failed = matches!(c.outcome, HostOutcome::Failed(_));
534            m.insert(metric::ANSWERED_RATE.into(), count(answered));
535            m.insert(
536                metric::ABSTAINED_RATE.into(),
537                count(c.outcome == HostOutcome::Abstained),
538            );
539            m.insert(metric::FAILED_RATE.into(), count(failed));
540            m.insert(metric::SCORED_RATE.into(), count(*scored));
541            // A failed case is outside the latency samples, so a flow
542            // cannot look faster by failing (decision 6).
543            if !failed {
544                let sample = match &c.latency_us {
545                    // One correctly rounded division (3.3.3).
546                    HostLatency::Us(us) => CaseMeasure::Sample {
547                        value: *us as f64 / 1000.0,
548                    },
549                    HostLatency::Unknown(u) => CaseMeasure::Unknown {
550                        reason: format!("latency unknown: {}", u.unknown),
551                    },
552                };
553                m.insert(metric::LATENCY_P50.into(), sample.clone());
554                m.insert(metric::LATENCY_P95.into(), sample);
555            }
556            // Every case, failed ones included: a failed case still spent.
557            let cost = match &c.cost {
558                None => CaseMeasure::Unknown {
559                    reason: COST_NOT_DECLARED.into(),
560                },
561                Some(HostCost::Unknown(u)) => CaseMeasure::Unknown {
562                    reason: format!("cost unknown: {}", u.unknown),
563                },
564                Some(HostCost::Value(d)) => match d.units().checked_mul(1000) {
565                    // 1000 times the cost, exact in Decimal, then one rounding.
566                    Some(units) => CaseMeasure::Mean {
567                        num: Decimal::from_units(units).to_f64_nearest(),
568                        den: 1,
569                    },
570                    None => CaseMeasure::Unknown {
571                        reason: "cost out of range".into(),
572                    },
573                },
574            };
575            m.insert(metric::COST_PER_1000.into(), cost);
576            HostCaseMeasures {
577                case: c.case.clone(),
578                outcome: c.outcome.clone(),
579                measures: m,
580            }
581        })
582        .collect();
583
584    let mut metrics: Vec<HostMetric> = vec![];
585    for name in metric::ALL {
586        metrics.push(HostMetric {
587            name: name.into(),
588            derivation: Derivation::ExactDerived,
589            opaque: false,
590            value: None,
591        });
592    }
593    for new in rubric
594        .into_iter()
595        .chain(agreement)
596        .chain(model_rubric)
597        .chain(model_agreement)
598    {
599        let (model, opaque) = names
600            .get(&new.metric.name)
601            .copied()
602            .unwrap_or((false, false));
603        for (c, m) in out.iter_mut().zip(new.cases) {
604            if let Some(m) = m {
605                c.measures.insert(new.metric.name.clone(), m);
606            }
607        }
608        metrics.push(HostMetric {
609            name: new.metric.name,
610            derivation: if model {
611                Derivation::ModelDerived
612            } else {
613                Derivation::ExactDerived
614            },
615            opaque,
616            value: Some(new.metric.value),
617        });
618    }
619    Ok(HostMeasured {
620        cases: HostCases {
621            schema: HOST_CASES.into(),
622            assertion: Assertion::Host,
623            host_run: record
624                .record_digest()
625                .map_err(|e| run_err(format!("the record has no record digest: {e}")))?,
626            cases: out,
627        },
628        metrics,
629    })
630}
631
632/// The form of a `host/` metric's measure (3.3.3); none for any other
633/// name.
634pub fn form(name: &str) -> Option<crate::repeat_stats::Form> {
635    use crate::repeat_stats::Form;
636    let quantile = |q: &str| Form::Sample {
637        quantile: Decimal::parse(q).expect("a fixed quantile"),
638    };
639    Some(match name {
640        metric::ANSWERED_RATE
641        | metric::ABSTAINED_RATE
642        | metric::FAILED_RATE
643        | metric::SCORED_RATE => Form::CountRatio,
644        metric::LATENCY_P50 => quantile("0.5"),
645        metric::LATENCY_P95 => quantile("0.95"),
646        metric::COST_PER_1000 => Form::Mean,
647        _ => return None,
648    })
649}
rustev_eval/host_run.rs

rustev_eval/
host_run.rs
