Skip to main content

evalhub_core/
badge.rs

1//! Badges: facts the hub can state about a version without judging it.
2//!
3//! A badge is a boolean derived from the record and the hub's own state,
4//! computed on ingest and stored with the version. Badges are how the hub
5//! says "this record is anchored" or "this harness is known" without ever
6//! saying "this result is correct". They are inputs to the reader's
7//! judgement, never a verdict.
8//!
9//! | Badge                | Condition                                                                           | Source of truth   |
10//! | -------------------- | ----------------------------------------------------------------------------------- | ----------------- |
11//! | `refs_resolved`      | every `relations[].to` resolved to a `version_id`                                   | store, at ingest  |
12//! | `harness_registered` | `harness.name@version` is a registry entry                                           | registry          |
13//! | `metric_registered`  | every `results[].metric` is a registry entry                                        | registry          |
14//! | `env_pinned`         | `env.git.dirty == false && env.git.commit && trial.sandbox.digest` all present      | record            |
15//! | `redacted`           | `redaction.applied == true`                                                         | record            |
16//!
17//! "Every" is read the usual way: a record with no relations has every
18//! relation resolved, and one with no results has every metric registered.
19//! The caller passes those facts in [`BadgeInput`]; the store computes them
20//! at ingest and is expected to report `true` for the vacuous cases.
21//!
22//! # Recomputation
23//!
24//! Badges derived from the record alone (`env_pinned`, `redacted`) never
25//! change, because the record never changes. Badges derived from the
26//! registry can: when a harness or metric is registered later, a job
27//! recomputes `harness_registered` / `metric_registered` for the versions
28//! that cite it. `refs_resolved` is not recomputed — a Card that cited an
29//! Eval before it existed keeps its unresolved status as a fact about
30//! publication order; the relation itself resolves lazily in the graph API.
31//!
32//! # Why there is no `verified` badge
33//!
34//! Because the hub did not run anything. Every badge names a fact the hub
35//! actually checked; "verified" would name a judgement it is not in a
36//! position to make.
37//!
38//! This module takes the record plus a small [`BadgeInput`] the server
39//! assembles (which references resolved, which registry entries exist) and
40//! returns the badge set. It does not query anything.
41
42use std::str::FromStr;
43
44use serde_json::Value;
45
46/// What the caller learned from the store, needed to decide the badges that
47/// depend on hub state rather than on the record alone.
48#[derive(Debug, Default, Clone)]
49pub struct BadgeInput {
50    /// `true` when every `relations[].to` in the record resolved to a stored
51    /// version (vacuously `true` when there are none).
52    pub all_refs_resolved: bool,
53    /// `true` when `harness.name@version` exists in the registry.
54    pub harness_registered: bool,
55    /// `true` when every `results[].metric` exists in the registry
56    /// (vacuously `true` when there are none).
57    pub all_metrics_registered: bool,
58}
59
60/// The badges the hub can award.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub enum Badge {
63    /// Every relation target resolved to a stored version at ingest.
64    RefsResolved,
65    /// The harness is a registry entry.
66    HarnessRegistered,
67    /// Every metric is a registry entry.
68    MetricRegistered,
69    /// The run is reproducible from the record: clean git tree with a
70    /// commit, and a sandbox image pinned by digest.
71    EnvPinned,
72    /// The producer states it redacted something before publishing.
73    Redacted,
74}
75
76impl Badge {
77    /// All badges, in the order [`compute`] reports them.
78    pub const ALL: [Badge; 5] = [
79        Badge::RefsResolved,
80        Badge::HarnessRegistered,
81        Badge::MetricRegistered,
82        Badge::EnvPinned,
83        Badge::Redacted,
84    ];
85
86    /// The badge's name as stored and served.
87    pub const fn as_str(self) -> &'static str {
88        match self {
89            Badge::RefsResolved => "refs_resolved",
90            Badge::HarnessRegistered => "harness_registered",
91            Badge::MetricRegistered => "metric_registered",
92            Badge::EnvPinned => "env_pinned",
93            Badge::Redacted => "redacted",
94        }
95    }
96}
97
98/// The text form was not a badge name.
99#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
100#[error("not a badge: {0:?}")]
101pub struct ParseBadgeError(String);
102
103impl FromStr for Badge {
104    type Err = ParseBadgeError;
105
106    fn from_str(s: &str) -> Result<Self, Self::Err> {
107        Badge::ALL
108            .into_iter()
109            .find(|b| b.as_str() == s)
110            .ok_or_else(|| ParseBadgeError(s.to_string()))
111    }
112}
113
114impl std::fmt::Display for Badge {
115    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
116        f.write_str(self.as_str())
117    }
118}
119
120/// Decide the badges of `value` given what the store reported in `input`.
121/// Returned in [`Badge::ALL`] order. Pure; never fails.
122pub fn compute(value: &Value, input: &BadgeInput) -> Vec<Badge> {
123    let mut out = Vec::new();
124    if input.all_refs_resolved {
125        out.push(Badge::RefsResolved);
126    }
127    if input.harness_registered {
128        out.push(Badge::HarnessRegistered);
129    }
130    if input.all_metrics_registered {
131        out.push(Badge::MetricRegistered);
132    }
133    if env_pinned(value) {
134        out.push(Badge::EnvPinned);
135    }
136    if value.pointer("/redaction/applied") == Some(&Value::Bool(true)) {
137        out.push(Badge::Redacted);
138    }
139    out
140}
141
142fn env_pinned(value: &Value) -> bool {
143    let non_empty = |p: &str| {
144        value
145            .pointer(p)
146            .and_then(Value::as_str)
147            .is_some_and(|s| !s.is_empty())
148    };
149    value.pointer("/env/git/dirty") == Some(&Value::Bool(false))
150        && non_empty("/env/git/commit")
151        && non_empty("/trial/sandbox/digest")
152}
153
154#[cfg(test)]
155mod tests {
156    #![allow(clippy::unwrap_used)]
157
158    use super::*;
159    use serde_json::json;
160
161    fn pinned() -> Value {
162        json!({
163            "env": {"git": {"commit": "abc", "dirty": false}},
164            "trial": {"sandbox": {"image": "x", "digest": "sha256:1"}},
165        })
166    }
167
168    #[test]
169    fn record_only_badges() {
170        let none = BadgeInput::default();
171        assert_eq!(compute(&pinned(), &none), vec![Badge::EnvPinned]);
172
173        let mut v = pinned();
174        v["env"]["git"]["dirty"] = json!(true);
175        assert_eq!(compute(&v, &none), vec![]);
176
177        let mut v = pinned();
178        v["env"]["git"]["commit"] = json!("");
179        assert_eq!(compute(&v, &none), vec![]);
180
181        let mut v = pinned();
182        v["trial"]["sandbox"]["digest"] = json!(null);
183        assert_eq!(compute(&v, &none), vec![]);
184
185        let v = json!({"redaction": {"applied": true}});
186        assert_eq!(compute(&v, &none), vec![Badge::Redacted]);
187        let v = json!({"redaction": {"applied": false}});
188        assert_eq!(compute(&v, &none), vec![]);
189    }
190
191    #[test]
192    fn input_badges_and_order() {
193        let input = BadgeInput {
194            all_refs_resolved: true,
195            harness_registered: true,
196            all_metrics_registered: true,
197        };
198        let mut v = pinned();
199        v["redaction"] = json!({"applied": true});
200        assert_eq!(compute(&v, &input), Badge::ALL.to_vec());
201        assert_eq!(
202            compute(&json!({}), &input),
203            vec![
204                Badge::RefsResolved,
205                Badge::HarnessRegistered,
206                Badge::MetricRegistered
207            ]
208        );
209    }
210
211    #[test]
212    fn names_round_trip() {
213        for b in Badge::ALL {
214            assert_eq!(b.as_str().parse::<Badge>().unwrap(), b);
215        }
216        assert!("verified".parse::<Badge>().is_err());
217    }
218}