evalhub_core/badge.rs
1//! Badges: facts the hub can state about a version without judging it.
2//!
3//! A badge is a boolean derived from the record and the hub's own state,
4//! computed on ingest and stored with the version. Badges are how the hub
5//! says "this record is anchored" or "this harness is known" without ever
6//! saying "this result is correct". They are inputs to the reader's
7//! judgement, never a verdict.
8//!
9//! | Badge | Condition | Source of truth |
10//! | -------------------- | ----------------------------------------------------------------------------------- | ----------------- |
11//! | `refs_resolved` | every `relations[].to` resolved to a `version_id` | store, at ingest |
12//! | `harness_registered` | `harness.name@version` is a registry entry | registry |
13//! | `metric_registered` | every `results[].metric` is a registry entry | registry |
14//! | `env_pinned` | `env.git.dirty == false && env.git.commit && trial.sandbox.digest` all present | record |
15//! | `redacted` | `redaction.applied == true` | record |
16//!
17//! "Every" is read the usual way: a record with no relations has every
18//! relation resolved, and one with no results has every metric registered.
19//! The caller passes those facts in [`BadgeInput`]; the store computes them
20//! at ingest and is expected to report `true` for the vacuous cases.
21//!
22//! # Recomputation
23//!
24//! Badges derived from the record alone (`env_pinned`, `redacted`) never
25//! change, because the record never changes. Badges derived from the
26//! registry can: when a harness or metric is registered later, a job
27//! recomputes `harness_registered` / `metric_registered` for the versions
28//! that cite it. `refs_resolved` is not recomputed — a Card that cited an
29//! Eval before it existed keeps its unresolved status as a fact about
30//! publication order; the relation itself resolves lazily in the graph API.
31//!
32//! # Why there is no `verified` badge
33//!
34//! Because the hub did not run anything. Every badge names a fact the hub
35//! actually checked; "verified" would name a judgement it is not in a
36//! position to make.
37//!
38//! This module takes the record plus a small [`BadgeInput`] the server
39//! assembles (which references resolved, which registry entries exist) and
40//! returns the badge set. It does not query anything.
41
42use std::str::FromStr;
43
44use serde_json::Value;
45
46/// What the caller learned from the store, needed to decide the badges that
47/// depend on hub state rather than on the record alone.
48#[derive(Debug, Default, Clone)]
49pub struct BadgeInput {
50 /// `true` when every `relations[].to` in the record resolved to a stored
51 /// version (vacuously `true` when there are none).
52 pub all_refs_resolved: bool,
53 /// `true` when `harness.name@version` exists in the registry.
54 pub harness_registered: bool,
55 /// `true` when every `results[].metric` exists in the registry
56 /// (vacuously `true` when there are none).
57 pub all_metrics_registered: bool,
58}
59
60/// The badges the hub can award.
61#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
62pub enum Badge {
63 /// Every relation target resolved to a stored version at ingest.
64 RefsResolved,
65 /// The harness is a registry entry.
66 HarnessRegistered,
67 /// Every metric is a registry entry.
68 MetricRegistered,
69 /// The run is reproducible from the record: clean git tree with a
70 /// commit, and a sandbox image pinned by digest.
71 EnvPinned,
72 /// The producer states it redacted something before publishing.
73 Redacted,
74}
75
76impl Badge {
77 /// All badges, in the order [`compute`] reports them.
78 pub const ALL: [Badge; 5] = [
79 Badge::RefsResolved,
80 Badge::HarnessRegistered,
81 Badge::MetricRegistered,
82 Badge::EnvPinned,
83 Badge::Redacted,
84 ];
85
86 /// The badge's name as stored and served.
87 pub const fn as_str(self) -> &'static str {
88 match self {
89 Badge::RefsResolved => "refs_resolved",
90 Badge::HarnessRegistered => "harness_registered",
91 Badge::MetricRegistered => "metric_registered",
92 Badge::EnvPinned => "env_pinned",
93 Badge::Redacted => "redacted",
94 }
95 }
96}
97
98/// The text form was not a badge name.
99#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
100#[error("not a badge: {0:?}")]
101pub struct ParseBadgeError(String);
102
103impl FromStr for Badge {
104 type Err = ParseBadgeError;
105
106 fn from_str(s: &str) -> Result<Self, Self::Err> {
107 Badge::ALL
108 .into_iter()
109 .find(|b| b.as_str() == s)
110 .ok_or_else(|| ParseBadgeError(s.to_string()))
111 }
112}
113
114impl std::fmt::Display for Badge {
115 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
116 f.write_str(self.as_str())
117 }
118}
119
120/// Decide the badges of `value` given what the store reported in `input`.
121/// Returned in [`Badge::ALL`] order. Pure; never fails.
122pub fn compute(value: &Value, input: &BadgeInput) -> Vec<Badge> {
123 let mut out = Vec::new();
124 if input.all_refs_resolved {
125 out.push(Badge::RefsResolved);
126 }
127 if input.harness_registered {
128 out.push(Badge::HarnessRegistered);
129 }
130 if input.all_metrics_registered {
131 out.push(Badge::MetricRegistered);
132 }
133 if env_pinned(value) {
134 out.push(Badge::EnvPinned);
135 }
136 if value.pointer("/redaction/applied") == Some(&Value::Bool(true)) {
137 out.push(Badge::Redacted);
138 }
139 out
140}
141
142fn env_pinned(value: &Value) -> bool {
143 let non_empty = |p: &str| {
144 value
145 .pointer(p)
146 .and_then(Value::as_str)
147 .is_some_and(|s| !s.is_empty())
148 };
149 value.pointer("/env/git/dirty") == Some(&Value::Bool(false))
150 && non_empty("/env/git/commit")
151 && non_empty("/trial/sandbox/digest")
152}
153
154#[cfg(test)]
155mod tests {
156 #![allow(clippy::unwrap_used)]
157
158 use super::*;
159 use serde_json::json;
160
161 fn pinned() -> Value {
162 json!({
163 "env": {"git": {"commit": "abc", "dirty": false}},
164 "trial": {"sandbox": {"image": "x", "digest": "sha256:1"}},
165 })
166 }
167
168 #[test]
169 fn record_only_badges() {
170 let none = BadgeInput::default();
171 assert_eq!(compute(&pinned(), &none), vec![Badge::EnvPinned]);
172
173 let mut v = pinned();
174 v["env"]["git"]["dirty"] = json!(true);
175 assert_eq!(compute(&v, &none), vec![]);
176
177 let mut v = pinned();
178 v["env"]["git"]["commit"] = json!("");
179 assert_eq!(compute(&v, &none), vec![]);
180
181 let mut v = pinned();
182 v["trial"]["sandbox"]["digest"] = json!(null);
183 assert_eq!(compute(&v, &none), vec![]);
184
185 let v = json!({"redaction": {"applied": true}});
186 assert_eq!(compute(&v, &none), vec![Badge::Redacted]);
187 let v = json!({"redaction": {"applied": false}});
188 assert_eq!(compute(&v, &none), vec![]);
189 }
190
191 #[test]
192 fn input_badges_and_order() {
193 let input = BadgeInput {
194 all_refs_resolved: true,
195 harness_registered: true,
196 all_metrics_registered: true,
197 };
198 let mut v = pinned();
199 v["redaction"] = json!({"applied": true});
200 assert_eq!(compute(&v, &input), Badge::ALL.to_vec());
201 assert_eq!(
202 compute(&json!({}), &input),
203 vec![
204 Badge::RefsResolved,
205 Badge::HarnessRegistered,
206 Badge::MetricRegistered
207 ]
208 );
209 }
210
211 #[test]
212 fn names_round_trip() {
213 for b in Badge::ALL {
214 assert_eq!(b.as_str().parse::<Badge>().unwrap(), b);
215 }
216 assert!("verified".parse::<Badge>().is_err());
217 }
218}