evalhub_core/registry.rs
1//! The `core/` registry: the vocabulary that ships with the hub.
2//!
3//! The registry is the hub's namespace-scoped, immutable catalogue of
4//! definitions: `kind/{ns}/{id}@{version}` for four kinds —
5//!
6//! | Kind | Defines | Used by |
7//! | ---------------- | ---------------------------------------------------------------- | ------------------------------------ |
8//! | `metrics` | `{ id, lower_is_better, description }` | `metric_registered` badge, sort direction |
9//! | `harnesses` | `{ name, version, homepage }` | `harness_registered` badge |
10//! | `relation_types` | `{ from, to, inverse }` | relation validation and traversal |
11//! | `ext_schemas` | `{ ns, schema, fingerprint: bool }` | typed `ext` indexing, fingerprint opt-in |
12//!
13//! Entries under `core/` are defined in this module and are read-only over
14//! the API. Everything else is written by namespaces with `write` scope via
15//! `PUT /registry/{kind}/{ns}/{id}@{version}` and stored in the `registry`
16//! table. The store seeds the `core/` rows from the constants here, so
17//! there is one definition of the vocabulary.
18//!
19//! # `core/` metrics (v0)
20//!
21//! `pass_rate`, `accuracy`, `f1`, `exact_match`, `pass_at_k`, `mean_score`,
22//! `latency_ms` (lower is better), `cost_usd` (lower is better). This is a
23//! starting vocabulary, not a taxonomy; producers register their own.
24//!
25//! # `core/` relation types
26//!
27//! | Type | From → To | Inverse |
28//! | -------------- | ------------ | ------------- |
29//! | `uses_eval` | Card → Eval | `used_by` |
30//! | `retry_of` | Card → Card | `retried_by` |
31//! | `rerun_of` | Card → Card | `rerun_by` |
32//! | `judged_by` | Card → Card | `judges` |
33//! | `baseline_of` | Card → Card | `compared_to` |
34//! | `derived_from` | Eval → Eval | `derives` |
35//! | `subset_of` | Eval → Eval | `superset_of` |
36//!
37//! # Immutability
38//!
39//! A registry entry is never edited. A change is a new `@{version}`. This
40//! matters for `ext_schemas` in particular: the fingerprints and index
41//! entries computed under one version of an extension schema stay valid,
42//! and the `fingerprints` table records which registry version they were
43//! computed under.
44//!
45//! # `ext_schemas` and the `applying` state
46//!
47//! Registering an `ext_schema` returns `202`: the entry is created in state
48//! `applying` and the store starts building the typed index for the new
49//! paths (see `evalhub_store::index`). While applying, those paths behave as
50//! unregistered (`eq` / `exists` only). When the index is valid the entry
51//! flips to `applied`. This module defines the entry shape and the state;
52//! the store owns the transition.
53
54use evalhub_schema::RecordKind;
55
56/// The version every `core/` entry is published under.
57pub const CORE_VERSION: &str = "1";
58
59/// A metric definition.
60#[derive(Debug, Clone, Copy, PartialEq, Eq)]
61pub struct MetricDef {
62 /// Registry id, `core/{name}`.
63 pub id: &'static str,
64 /// Whether a smaller value is the better one (sort direction).
65 pub lower_is_better: bool,
66 /// One line for a reader.
67 pub description: &'static str,
68}
69
70/// The metrics that ship with the hub.
71pub const CORE_METRICS: &[MetricDef] = &[
72 MetricDef {
73 id: "core/pass_rate",
74 lower_is_better: false,
75 description: "Fraction of items graded as passing.",
76 },
77 MetricDef {
78 id: "core/accuracy",
79 lower_is_better: false,
80 description: "Fraction of items answered correctly.",
81 },
82 MetricDef {
83 id: "core/f1",
84 lower_is_better: false,
85 description: "Harmonic mean of precision and recall.",
86 },
87 MetricDef {
88 id: "core/exact_match",
89 lower_is_better: false,
90 description: "Fraction of items whose output matched the reference exactly.",
91 },
92 MetricDef {
93 id: "core/pass_at_k",
94 lower_is_better: false,
95 description: "Probability that at least one of k samples passes.",
96 },
97 MetricDef {
98 id: "core/mean_score",
99 lower_is_better: false,
100 description: "Mean of a per-item score on the producer's scale.",
101 },
102 MetricDef {
103 id: "core/latency_ms",
104 lower_is_better: true,
105 description: "Wall-clock time per item, in milliseconds.",
106 },
107 MetricDef {
108 id: "core/cost_usd",
109 lower_is_better: true,
110 description: "Cost per item, in US dollars.",
111 },
112];
113
114/// A relation type definition.
115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
116pub struct RelationTypeDef {
117 /// Registry id, `core/{name}`.
118 pub id: &'static str,
119 /// Kind of the record the edge starts from.
120 pub from: RecordKind,
121 /// Kind of the record the edge points at.
122 pub to: RecordKind,
123 /// Name of the edge read backwards.
124 pub inverse: &'static str,
125}
126
127/// The relation types that ship with the hub.
128pub const CORE_RELATION_TYPES: &[RelationTypeDef] = &[
129 RelationTypeDef {
130 id: "core/uses_eval",
131 from: RecordKind::Card,
132 to: RecordKind::Eval,
133 inverse: "used_by",
134 },
135 RelationTypeDef {
136 id: "core/retry_of",
137 from: RecordKind::Card,
138 to: RecordKind::Card,
139 inverse: "retried_by",
140 },
141 RelationTypeDef {
142 id: "core/rerun_of",
143 from: RecordKind::Card,
144 to: RecordKind::Card,
145 inverse: "rerun_by",
146 },
147 RelationTypeDef {
148 id: "core/judged_by",
149 from: RecordKind::Card,
150 to: RecordKind::Card,
151 inverse: "judges",
152 },
153 RelationTypeDef {
154 id: "core/baseline_of",
155 from: RecordKind::Card,
156 to: RecordKind::Card,
157 inverse: "compared_to",
158 },
159 RelationTypeDef {
160 id: "core/derived_from",
161 from: RecordKind::Eval,
162 to: RecordKind::Eval,
163 inverse: "derives",
164 },
165 RelationTypeDef {
166 id: "core/subset_of",
167 from: RecordKind::Eval,
168 to: RecordKind::Eval,
169 inverse: "superset_of",
170 },
171];
172
173/// Look a `core/` metric up by id.
174pub fn core_metric(id: &str) -> Option<&'static MetricDef> {
175 CORE_METRICS.iter().find(|m| m.id == id)
176}
177
178/// Look a `core/` relation type up by id.
179pub fn core_relation_type(id: &str) -> Option<&'static RelationTypeDef> {
180 CORE_RELATION_TYPES.iter().find(|r| r.id == id)
181}
182
183#[cfg(test)]
184mod tests {
185 #![allow(clippy::unwrap_used)]
186
187 use super::*;
188 use crate::validate::is_valid_id;
189
190 #[test]
191 fn ids_are_well_formed_and_in_core() {
192 for m in CORE_METRICS {
193 assert!(is_valid_id(m.id), "{}", m.id);
194 assert!(m.id.starts_with("core/"), "{}", m.id);
195 }
196 for r in CORE_RELATION_TYPES {
197 assert!(is_valid_id(r.id), "{}", r.id);
198 assert!(r.id.starts_with("core/"), "{}", r.id);
199 assert!(!r.inverse.is_empty());
200 }
201 assert_eq!(CORE_METRICS.len(), 8);
202 assert_eq!(CORE_RELATION_TYPES.len(), 7);
203 }
204
205 #[test]
206 fn lookups() {
207 assert!(core_metric("core/latency_ms").is_some_and(|m| m.lower_is_better));
208 assert!(core_metric("core/pass_rate").is_some_and(|m| !m.lower_is_better));
209 assert!(core_metric("alice/x").is_none());
210 let r = core_relation_type("core/uses_eval").unwrap();
211 assert_eq!(
212 (r.from, r.to, r.inverse),
213 (RecordKind::Card, RecordKind::Eval, "used_by")
214 );
215 assert!(core_relation_type("core/nope").is_none());
216 }
217}