Skip to main content

evalhub_schema/
card.rs

1//! The Card record: what was measured, how, and what the score was.
2//!
3//! A Card is the unit a harness publishes after an evaluation. It is a claim
4//! made by the producer, kept verbatim by the hub. The hub adds an id, a
5//! sequence number, a content hash, badges and fingerprints around it, but
6//! never edits the record itself; a correction is a new version.
7//!
8//! # Shape
9//!
10//! ```text
11//! Card
12//! ├── schema        "evalhub.card/1.1" (or "evalhub.card/1.0")
13//! ├── title         free text
14//! ├── producer      { name, version }            who wrote this record
15//! ├── model / task / harness / generation / trial / grading / env
16//! │                 seven facets (see `facet`)   what was measured, under what conditions
17//! ├── results[]     { metric, value, n, aggregation, uncertainty, by, samples_ref }
18//! ├── counts        { attempted, completed, failed, skipped, errored }
19//! ├── run_results[] { eval, run_id, metric, value?, label?, by?, from? }
20//! │                 per-run verdicts on the runs of a cited Eval (1.1)
21//! ├── relations[]   { type, to, attrs }          edges to other records
22//! ├── attachments[] { path, sha256, size, media_type }
23//! ├── redaction     { applied, method, fields }
24//! ├── provenance    { source_type, evaluator_relationship }
25//! └── ext           { "{ns}/{name}": { ... } }   producer-private extension
26//! ```
27//!
28//! A complete example:
29//!
30//! ```json
31//! {
32//!   "schema": "evalhub.card/1.1",
33//!   "title": "qwen3.6-32b on single2 K4",
34//!   "producer": {"name": "my-harness", "version": "0.4.1"},
35//!   "model": {"id": "qwen3.6-32b", "provider": "local", "quantization": {"method": "awq", "dtype": "int4"}},
36//!   "task": {"id": "single2", "version": "3", "split": "test", "n": 33, "seed": 7},
37//!   "harness": {"name": "my-harness", "version": "0.4.1", "prompt_template_sha256": "…"},
38//!   "generation": {"temperature": 0.0, "max_tokens": 4096},
39//!   "trial": {"k": 4, "pass_at_k": true, "max_turns": 20, "sandbox": {"image": "ghcr.io/x/y", "digest": "sha256:…"}},
40//!   "grading": {"graders": [{"name": "exact_match", "kind": "deterministic"}]},
41//!   "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
42//!   "results": [{
43//!     "metric": "core/pass_rate",
44//!     "value": 0.62, "n": 33,
45//!     "aggregation": "mean",
46//!     "uncertainty": {"stderr": 0.08, "ci": {"level": 0.95, "low": 0.45, "high": 0.77}},
47//!     "by": {"grader": "exact_match"},
48//!     "samples_ref": "samples.jsonl"
49//!   }],
50//!   "counts": {"attempted": 33, "completed": 30, "failed": 2, "skipped": 1, "errored": 0},
51//!   "relations": [{"type": "core/uses_eval", "to": "alice/single2-k4@3"}],
52//!   "attachments": [{"path": "samples.jsonl", "sha256": "…", "size": 12345, "media_type": "application/x-ndjson"}],
53//!   "redaction": {"applied": true, "method": "regex+llm", "fields": ["samples.jsonl:response"]},
54//!   "provenance": {"source_type": "evaluation_run", "evaluator_relationship": "first_party"},
55//!   "ext": {"alice/qwen-loop": {"rung": 4}}
56//! }
57//! ```
58//!
59//! # Results
60//!
61//! `results[].metric` is a registry id in the form `{ns}/{name}`
62//! (`core/pass_rate`). A metric the registry does not know is still accepted
63//! if it has that form; a bare name is rejected (`metric_id_invalid`). Whether
64//! the metric is registered is surfaced as the `metric_registered` badge, not
65//! as an error — the hub records claims, it does not gate on vocabulary.
66//!
67//! `aggregation` names how `value` was derived from the samples (`mean`,
68//! `pass_at_k`, `median`, `sum`, `custom`). `uncertainty` carries whatever the
69//! producer computed (standard error, a confidence interval). `by` records the
70//! grader or other partition the number applies to. `samples_ref` names an
71//! entry in `attachments[]` holding the per-sample rows; the hub checks that
72//! the path exists in `attachments[]` and nothing else about it.
73//!
74//! # Counts
75//!
76//! If `results` is non-empty, `counts` is required and must satisfy
77//! `attempted >= completed + failed + skipped + errored`. This is the one
78//! arithmetic rule the hub enforces, because a score without the denominator
79//! it was computed over is not comparable to anything.
80//!
81//! # Run results
82//!
83//! ```text
84//! relations[core/uses_eval] { to: "{ns}/{name}@{seq}", attrs: { runs?: [run_id] } }
85//! run_results[]             { eval: "{ns}/{name}", run_id, metric, value?, label?, by?, from? }
86//! ```
87//!
88//! `results[]` is the aggregate; `run_results[]` is the verdict per run
89//! that the aggregate was computed from. A run of an Eval records what
90//! happened and what was measured directly (`crate::run`); it never says
91//! whether it passed. That judgement depends on a grader, so it is the
92//! Card's claim and lives here: one element per `(run, metric)` the Card
93//! judged, with a number (`value`), a label (`label`, such as `pass` or
94//! `fail`), or both. At least one of the two is present
95//! (`run_result_value_or_label`). `by` partitions the verdict the way
96//! `results[].by` does, and `from` optionally names the run's `metrics`
97//! keys the verdict was derived from. The hub does not recompute anything
98//! and does not check that `results[]` agrees with `run_results[]`; how the
99//! verdicts were produced is the grading facet's to describe.
100//!
101//! Re-grading appends a Card version; the earlier verdicts stay with the
102//! earlier version. Grading the same runs with a different grader is a
103//! different Card.
104//!
105//! `run_results` was added in `evalhub.card/1.1`. It is an optional key, so
106//! the bump is minor: one schema document serves both identifiers, a Card
107//! declaring `evalhub.card/1.0` may carry `run_results` too, and Cards
108//! stored under 1.0 are not rewritten.
109//!
110//! ## The used set
111//!
112//! Which runs a Card used is decided per cited Eval record, and that set of
113//! `run_id`s is the Card's *used set* for the record. For each Eval record,
114//! it is the union of `attrs.runs` over the Card's `core/uses_eval`
115//! relations that resolve to that record; or, when none of those relations
116//! carries `attrs.runs`, every run of the record that is neither archived
117//! nor deleted at the moment the Card is posted. The hub fixes the used set
118//! when the Card is posted and remembers each run's content hash with it,
119//! which is how a later change to a run is detected.
120//!
121//! Against the used set:
122//!
123//! - `run_results[].eval` names the record (`{ns}/{name}`, no `@seq`) of
124//!   one of the Card's `core/uses_eval` relations
125//!   (`run_results_eval_unknown`).
126//! - every `run_id` in a used set has a row in that Eval, archived and
127//!   deleted rows included (`run_unknown`). A run is always in the hub, so
128//!   a missing row is a mistake, not an unresolved reference, and an Eval
129//!   the writer may not see answers the same way as a run that does not
130//!   exist.
131//! - every `run_results[].run_id` is in the used set of its `eval`
132//!   (`run_not_in_used_set`).
133//!
134//! # Provenance and redaction
135//!
136//! `provenance` says where the numbers came from (`evaluation_run`,
137//! `paper`, `model_card`, …) and who ran it relative to the model
138//! (`first_party`, `third_party`). `redaction` says whether the producer
139//! removed anything before publishing, by what method, and from which fields.
140//! The hub does not redact; it only makes the producer's statement visible
141//! and searchable, and awards the `redacted` badge when `applied` is true.
142//!
143//! # Fields the hub owns
144//!
145//! `id`, `version_id`, `seq`, `label`, `content_hash`, `created_at`,
146//! `changed[]`, `badges[]` and the per-facet fingerprints are not part of
147//! this type. They are returned alongside it in API responses and are
148//! documented in `evalhub_server::api::records`.
149
150use schemars::JsonSchema;
151use serde::{Deserialize, Serialize};
152
153use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
154use crate::facet::{Env, Generation, Grading, Harness, Model, Task, Trial};
155
156/// A Card: what was measured, how, and what the score was.
157#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
158#[serde(deny_unknown_fields)]
159pub struct Card {
160    /// Schema identifier: `evalhub.card/1.1`, or `evalhub.card/1.0`, which is
161    /// still accepted. Both are described by this one schema.
162    #[schemars(extend("enum" = ["evalhub.card/1.0", "evalhub.card/1.1"]))]
163    pub schema: String,
164    /// Human-readable title.
165    pub title: String,
166    /// The software that wrote this record.
167    pub producer: Producer,
168    /// The model facet.
169    #[serde(default, skip_serializing_if = "Option::is_none")]
170    pub model: Option<Model>,
171    /// The task facet.
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub task: Option<Task>,
174    /// The harness facet.
175    #[serde(default, skip_serializing_if = "Option::is_none")]
176    pub harness: Option<Harness>,
177    /// The generation facet.
178    #[serde(default, skip_serializing_if = "Option::is_none")]
179    pub generation: Option<Generation>,
180    /// The trial facet.
181    #[serde(default, skip_serializing_if = "Option::is_none")]
182    pub trial: Option<Trial>,
183    /// The grading facet.
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    pub grading: Option<Grading>,
186    /// The environment facet.
187    #[serde(default, skip_serializing_if = "Option::is_none")]
188    pub env: Option<Env>,
189    /// The scores.
190    #[serde(default, skip_serializing_if = "Vec::is_empty")]
191    pub results: Vec<ResultEntry>,
192    /// Item counts the scores were computed over. Required when `results` is non-empty.
193    #[serde(default, skip_serializing_if = "Option::is_none")]
194    pub counts: Option<Counts>,
195    /// Per-run verdicts on the runs of the Evals this Card uses
196    /// (`core/uses_eval`). Each `run_id` must be in the Card's used set for
197    /// its `eval`. Added in `evalhub.card/1.1`.
198    #[serde(default, skip_serializing_if = "Vec::is_empty")]
199    pub run_results: Vec<RunResult>,
200    /// Edges to other records.
201    #[serde(default, skip_serializing_if = "Vec::is_empty")]
202    pub relations: Vec<Relation>,
203    /// Files this record refers to.
204    #[serde(default, skip_serializing_if = "Vec::is_empty")]
205    pub attachments: Vec<Attachment>,
206    /// What the producer removed before publishing.
207    #[serde(default, skip_serializing_if = "Option::is_none")]
208    pub redaction: Option<Redaction>,
209    /// Where the numbers came from and who ran the evaluation.
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub provenance: Option<Provenance>,
212    /// Producer-private extensions keyed by namespace.
213    #[serde(default)]
214    pub ext: Ext,
215}
216
217/// One score.
218#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
219#[serde(deny_unknown_fields)]
220pub struct ResultEntry {
221    /// Metric as a registry id, `{ns}/{name}` (for example `core/pass_rate`).
222    pub metric: String,
223    /// The score.
224    pub value: f64,
225    /// Number of samples the score was computed over.
226    #[serde(default, skip_serializing_if = "Option::is_none")]
227    pub n: Option<u64>,
228    /// How `value` was derived from the samples.
229    pub aggregation: Aggregation,
230    /// Uncertainty the producer computed.
231    #[serde(default, skip_serializing_if = "Option::is_none")]
232    pub uncertainty: Option<Uncertainty>,
233    /// The partition this score applies to (for example which grader).
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub by: Option<serde_json::Map<String, serde_json::Value>>,
236    /// `attachments[].path` of the per-sample rows this score summarises.
237    #[serde(default, skip_serializing_if = "Option::is_none")]
238    pub samples_ref: Option<String>,
239}
240
241/// The Card's verdict on one run for one metric.
242#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
243#[serde(deny_unknown_fields)]
244pub struct RunResult {
245    /// The Eval record the run belongs to, `{ns}/{name}` (no `@seq`: runs
246    /// belong to the record, not to a version). Must be the record of one of
247    /// this Card's `core/uses_eval` relations.
248    pub eval: String,
249    /// The run judged. Must be in this Card's used set for `eval`.
250    pub run_id: String,
251    /// Metric as a registry id, `{ns}/{name}` (for example `core/pass`).
252    pub metric: String,
253    /// The verdict as a number. At least one of `value` and `label` is present.
254    #[serde(default, skip_serializing_if = "Option::is_none")]
255    pub value: Option<f64>,
256    /// The verdict as a label (for example `pass`, `fail`). At least one of
257    /// `value` and `label` is present.
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub label: Option<String>,
260    /// The partition this verdict applies to (for example which grader).
261    #[serde(default, skip_serializing_if = "Option::is_none")]
262    pub by: Option<serde_json::Map<String, serde_json::Value>>,
263    /// Keys of the run's `metrics` the verdict was derived from. Informational;
264    /// the hub does not recompute.
265    #[serde(default, skip_serializing_if = "Vec::is_empty")]
266    pub from: Vec<String>,
267}
268
269/// How a score was derived from its samples.
270#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
271#[serde(rename_all = "snake_case")]
272pub enum Aggregation {
273    /// Arithmetic mean over samples.
274    Mean,
275    /// pass@k over attempts.
276    PassAtK,
277    /// Median over samples.
278    Median,
279    /// Sum over samples.
280    Sum,
281    /// A producer-defined aggregation; describe it in `ext`.
282    Custom,
283}
284
285/// Uncertainty of a score.
286#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
287#[serde(deny_unknown_fields)]
288pub struct Uncertainty {
289    /// Standard error of the score.
290    #[serde(default, skip_serializing_if = "Option::is_none")]
291    pub stderr: Option<f64>,
292    /// Confidence interval of the score.
293    #[serde(default, skip_serializing_if = "Option::is_none")]
294    pub ci: Option<ConfidenceInterval>,
295}
296
297/// A confidence interval.
298#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
299#[serde(deny_unknown_fields)]
300pub struct ConfidenceInterval {
301    /// Confidence level, for example `0.95`.
302    pub level: f64,
303    /// Lower bound.
304    pub low: f64,
305    /// Upper bound.
306    pub high: f64,
307}
308
309/// Item counts. `attempted >= completed + failed + skipped + errored`.
310#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
311#[serde(deny_unknown_fields)]
312pub struct Counts {
313    /// Items the evaluation set out to run.
314    pub attempted: u64,
315    /// Items that produced a gradable response.
316    #[serde(default)]
317    pub completed: u64,
318    /// Items that were graded as failing.
319    #[serde(default)]
320    pub failed: u64,
321    /// Items skipped by the harness.
322    #[serde(default)]
323    pub skipped: u64,
324    /// Items that errored before grading.
325    #[serde(default)]
326    pub errored: u64,
327}
328
329/// Where the numbers came from.
330#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
331#[serde(deny_unknown_fields)]
332pub struct Provenance {
333    /// What kind of source the numbers were taken from.
334    pub source_type: SourceType,
335    /// Who ran the evaluation relative to the model's maker.
336    #[serde(default, skip_serializing_if = "Option::is_none")]
337    pub evaluator_relationship: Option<EvaluatorRelationship>,
338}
339
340/// The kind of source a Card's numbers were taken from.
341#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
342#[serde(rename_all = "snake_case")]
343pub enum SourceType {
344    /// The producer ran the evaluation and recorded the result.
345    EvaluationRun,
346    /// Transcribed from a paper.
347    Paper,
348    /// Transcribed from a model card.
349    ModelCard,
350    /// Transcribed from a leaderboard.
351    Leaderboard,
352    /// Something else; describe it in `ext`.
353    Other,
354}
355
356/// Who ran the evaluation relative to the model's maker.
357#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
358#[serde(rename_all = "snake_case")]
359pub enum EvaluatorRelationship {
360    /// The model's maker evaluated its own model.
361    FirstParty,
362    /// Someone else evaluated it.
363    ThirdParty,
364}