Skip to main content

evalhub_core/
registry.rs

1//! The `core/` registry: the vocabulary that ships with the hub.
2//!
3//! The registry is the hub's namespace-scoped, immutable catalogue of
4//! definitions: `kind/{ns}/{id}@{version}` for four kinds —
5//!
6//! | Kind             | Defines                                                          | Used by                              |
7//! | ---------------- | ---------------------------------------------------------------- | ------------------------------------ |
8//! | `metrics`        | `{ id, lower_is_better, description }`                           | `metric_registered` badge, sort direction |
9//! | `harnesses`      | `{ name, version, homepage }`                                    | `harness_registered` badge           |
10//! | `relation_types` | `{ from, to, inverse }`                                          | relation validation and traversal    |
11//! | `ext_schemas`    | `{ ns, schema, fingerprint: bool }`                              | typed `ext` indexing, fingerprint opt-in |
12//!
13//! Entries under `core/` are defined in this module and are read-only over
14//! the API. Everything else is written by namespaces with `write` scope via
15//! `PUT /registry/{kind}/{ns}/{id}@{version}` and stored in the `registry`
16//! table. The store seeds the `core/` rows from the constants here, so
17//! there is one definition of the vocabulary.
18//!
19//! # `core/` metrics (v0)
20//!
21//! `pass_rate`, `accuracy`, `f1`, `exact_match`, `pass_at_k`, `mean_score`,
22//! `latency_ms` (lower is better), `cost_usd` (lower is better). This is a
23//! starting vocabulary, not a taxonomy; producers register their own.
24//!
25//! # `core/` relation types
26//!
27//! | Type           | From → To    | Inverse       |
28//! | -------------- | ------------ | ------------- |
29//! | `uses_eval`    | Card → Eval  | `used_by`     |
30//! | `retry_of`     | Card → Card  | `retried_by`  |
31//! | `rerun_of`     | Card → Card  | `rerun_by`    |
32//! | `judged_by`    | Card → Card  | `judges`      |
33//! | `baseline_of`  | Card → Card  | `compared_to` |
34//! | `derived_from` | Eval → Eval  | `derives`     |
35//! | `subset_of`    | Eval → Eval  | `superset_of` |
36//!
37//! # Immutability
38//!
39//! A registry entry is never edited. A change is a new `@{version}`. This
40//! matters for `ext_schemas` in particular: the fingerprints and index
41//! entries computed under one version of an extension schema stay valid,
42//! and the `fingerprints` table records which registry version they were
43//! computed under.
44//!
45//! # `ext_schemas` and the `applying` state
46//!
47//! Registering an `ext_schema` returns `202`: the entry is created in state
48//! `applying` and the store starts building the typed index for the new
49//! paths (see `evalhub_store::index`). While applying, those paths behave as
50//! unregistered (`eq` / `exists` only). When the index is valid the entry
51//! flips to `applied`. This module defines the entry shape and the state;
52//! the store owns the transition.
53
54use evalhub_schema::RecordKind;
55
56/// The version every `core/` entry is published under.
57pub const CORE_VERSION: &str = "1";
58
59/// A metric definition.
60#[derive(Debug, Clone, Copy, PartialEq, Eq)]
61pub struct MetricDef {
62    /// Registry id, `core/{name}`.
63    pub id: &'static str,
64    /// Whether a smaller value is the better one (sort direction).
65    pub lower_is_better: bool,
66    /// One line for a reader.
67    pub description: &'static str,
68}
69
70/// The metrics that ship with the hub.
71pub const CORE_METRICS: &[MetricDef] = &[
72    MetricDef {
73        id: "core/pass_rate",
74        lower_is_better: false,
75        description: "Fraction of items graded as passing.",
76    },
77    MetricDef {
78        id: "core/accuracy",
79        lower_is_better: false,
80        description: "Fraction of items answered correctly.",
81    },
82    MetricDef {
83        id: "core/f1",
84        lower_is_better: false,
85        description: "Harmonic mean of precision and recall.",
86    },
87    MetricDef {
88        id: "core/exact_match",
89        lower_is_better: false,
90        description: "Fraction of items whose output matched the reference exactly.",
91    },
92    MetricDef {
93        id: "core/pass_at_k",
94        lower_is_better: false,
95        description: "Probability that at least one of k samples passes.",
96    },
97    MetricDef {
98        id: "core/mean_score",
99        lower_is_better: false,
100        description: "Mean of a per-item score on the producer's scale.",
101    },
102    MetricDef {
103        id: "core/latency_ms",
104        lower_is_better: true,
105        description: "Wall-clock time per item, in milliseconds.",
106    },
107    MetricDef {
108        id: "core/cost_usd",
109        lower_is_better: true,
110        description: "Cost per item, in US dollars.",
111    },
112];
113
114/// A relation type definition.
115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
116pub struct RelationTypeDef {
117    /// Registry id, `core/{name}`.
118    pub id: &'static str,
119    /// Kind of the record the edge starts from.
120    pub from: RecordKind,
121    /// Kind of the record the edge points at.
122    pub to: RecordKind,
123    /// Name of the edge read backwards.
124    pub inverse: &'static str,
125}
126
127/// The relation types that ship with the hub.
128pub const CORE_RELATION_TYPES: &[RelationTypeDef] = &[
129    RelationTypeDef {
130        id: "core/uses_eval",
131        from: RecordKind::Card,
132        to: RecordKind::Eval,
133        inverse: "used_by",
134    },
135    RelationTypeDef {
136        id: "core/retry_of",
137        from: RecordKind::Card,
138        to: RecordKind::Card,
139        inverse: "retried_by",
140    },
141    RelationTypeDef {
142        id: "core/rerun_of",
143        from: RecordKind::Card,
144        to: RecordKind::Card,
145        inverse: "rerun_by",
146    },
147    RelationTypeDef {
148        id: "core/judged_by",
149        from: RecordKind::Card,
150        to: RecordKind::Card,
151        inverse: "judges",
152    },
153    RelationTypeDef {
154        id: "core/baseline_of",
155        from: RecordKind::Card,
156        to: RecordKind::Card,
157        inverse: "compared_to",
158    },
159    RelationTypeDef {
160        id: "core/derived_from",
161        from: RecordKind::Eval,
162        to: RecordKind::Eval,
163        inverse: "derives",
164    },
165    RelationTypeDef {
166        id: "core/subset_of",
167        from: RecordKind::Eval,
168        to: RecordKind::Eval,
169        inverse: "superset_of",
170    },
171];
172
173/// Look a `core/` metric up by id.
174pub fn core_metric(id: &str) -> Option<&'static MetricDef> {
175    CORE_METRICS.iter().find(|m| m.id == id)
176}
177
178/// Look a `core/` relation type up by id.
179pub fn core_relation_type(id: &str) -> Option<&'static RelationTypeDef> {
180    CORE_RELATION_TYPES.iter().find(|r| r.id == id)
181}
182
183#[cfg(test)]
184mod tests {
185    #![allow(clippy::unwrap_used)]
186
187    use super::*;
188    use crate::validate::is_valid_id;
189
190    #[test]
191    fn ids_are_well_formed_and_in_core() {
192        for m in CORE_METRICS {
193            assert!(is_valid_id(m.id), "{}", m.id);
194            assert!(m.id.starts_with("core/"), "{}", m.id);
195        }
196        for r in CORE_RELATION_TYPES {
197            assert!(is_valid_id(r.id), "{}", r.id);
198            assert!(r.id.starts_with("core/"), "{}", r.id);
199            assert!(!r.inverse.is_empty());
200        }
201        assert_eq!(CORE_METRICS.len(), 8);
202        assert_eq!(CORE_RELATION_TYPES.len(), 7);
203    }
204
205    #[test]
206    fn lookups() {
207        assert!(core_metric("core/latency_ms").is_some_and(|m| m.lower_is_better));
208        assert!(core_metric("core/pass_rate").is_some_and(|m| !m.lower_is_better));
209        assert!(core_metric("alice/x").is_none());
210        let r = core_relation_type("core/uses_eval").unwrap();
211        assert_eq!(
212            (r.from, r.to, r.inverse),
213            (RecordKind::Card, RecordKind::Eval, "used_by")
214        );
215        assert!(core_relation_type("core/nope").is_none());
216    }
217}