evalhub_schema/card.rs
1//! The Card record: what was measured, how, and what the score was.
2//!
3//! A Card is the unit a harness publishes after an evaluation. It is a claim
4//! made by the producer, kept verbatim by the hub. The hub adds an id, a
5//! sequence number, a content hash, badges and fingerprints around it, but
6//! never edits the record itself; a correction is a new version.
7//!
8//! # Shape
9//!
10//! ```text
11//! Card
12//! ├── schema "evalhub.card/1.1" (or "evalhub.card/1.0")
13//! ├── title free text
14//! ├── producer { name, version } who wrote this record
15//! ├── model / task / harness / generation / trial / grading / env
16//! │ seven facets (see `facet`) what was measured, under what conditions
17//! ├── results[] { metric, value, n, aggregation, uncertainty, by, samples_ref }
18//! ├── counts { attempted, completed, failed, skipped, errored }
19//! ├── run_results[] { eval, run_id, metric, value?, label?, by?, from? }
20//! │ per-run verdicts on the runs of a cited Eval (1.1)
21//! ├── relations[] { type, to, attrs } edges to other records
22//! ├── attachments[] { path, sha256, size, media_type }
23//! ├── redaction { applied, method, fields }
24//! ├── provenance { source_type, evaluator_relationship }
25//! └── ext { "{ns}/{name}": { ... } } producer-private extension
26//! ```
27//!
28//! A complete example:
29//!
30//! ```json
31//! {
32//! "schema": "evalhub.card/1.1",
33//! "title": "qwen3.6-32b on single2 K4",
34//! "producer": {"name": "my-harness", "version": "0.4.1"},
35//! "model": {"id": "qwen3.6-32b", "provider": "local", "quantization": {"method": "awq", "dtype": "int4"}},
36//! "task": {"id": "single2", "version": "3", "split": "test", "n": 33, "seed": 7},
37//! "harness": {"name": "my-harness", "version": "0.4.1", "prompt_template_sha256": "…"},
38//! "generation": {"temperature": 0.0, "max_tokens": 4096},
39//! "trial": {"k": 4, "pass_at_k": true, "max_turns": 20, "sandbox": {"image": "ghcr.io/x/y", "digest": "sha256:…"}},
40//! "grading": {"graders": [{"name": "exact_match", "kind": "deterministic"}]},
41//! "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
42//! "results": [{
43//! "metric": "core/pass_rate",
44//! "value": 0.62, "n": 33,
45//! "aggregation": "mean",
46//! "uncertainty": {"stderr": 0.08, "ci": {"level": 0.95, "low": 0.45, "high": 0.77}},
47//! "by": {"grader": "exact_match"},
48//! "samples_ref": "samples.jsonl"
49//! }],
50//! "counts": {"attempted": 33, "completed": 30, "failed": 2, "skipped": 1, "errored": 0},
51//! "relations": [{"type": "core/uses_eval", "to": "alice/single2-k4@3"}],
52//! "attachments": [{"path": "samples.jsonl", "sha256": "…", "size": 12345, "media_type": "application/x-ndjson"}],
53//! "redaction": {"applied": true, "method": "regex+llm", "fields": ["samples.jsonl:response"]},
54//! "provenance": {"source_type": "evaluation_run", "evaluator_relationship": "first_party"},
55//! "ext": {"alice/qwen-loop": {"rung": 4}}
56//! }
57//! ```
58//!
59//! # Results
60//!
61//! `results[].metric` is a registry id in the form `{ns}/{name}`
62//! (`core/pass_rate`). A metric the registry does not know is still accepted
63//! if it has that form; a bare name is rejected (`metric_id_invalid`). Whether
64//! the metric is registered is surfaced as the `metric_registered` badge, not
65//! as an error — the hub records claims, it does not gate on vocabulary.
66//!
67//! `aggregation` names how `value` was derived from the samples (`mean`,
68//! `pass_at_k`, `median`, `sum`, `custom`). `uncertainty` carries whatever the
69//! producer computed (standard error, a confidence interval). `by` records the
70//! grader or other partition the number applies to. `samples_ref` names an
71//! entry in `attachments[]` holding the per-sample rows; the hub checks that
72//! the path exists in `attachments[]` and nothing else about it.
73//!
74//! # Counts
75//!
76//! If `results` is non-empty, `counts` is required and must satisfy
77//! `attempted >= completed + failed + skipped + errored`. This is the one
78//! arithmetic rule the hub enforces, because a score without the denominator
79//! it was computed over is not comparable to anything.
80//!
81//! # Run results
82//!
83//! ```text
84//! relations[core/uses_eval] { to: "{ns}/{name}@{seq}", attrs: { runs?: [run_id] } }
85//! run_results[] { eval: "{ns}/{name}", run_id, metric, value?, label?, by?, from? }
86//! ```
87//!
88//! `results[]` is the aggregate; `run_results[]` is the verdict per run
89//! that the aggregate was computed from. A run of an Eval records what
90//! happened and what was measured directly (`crate::run`); it never says
91//! whether it passed. That judgement depends on a grader, so it is the
92//! Card's claim and lives here: one element per `(run, metric)` the Card
93//! judged, with a number (`value`), a label (`label`, such as `pass` or
94//! `fail`), or both. At least one of the two is present
95//! (`run_result_value_or_label`). `by` partitions the verdict the way
96//! `results[].by` does, and `from` optionally names the run's `metrics`
97//! keys the verdict was derived from. The hub does not recompute anything
98//! and does not check that `results[]` agrees with `run_results[]`; how the
99//! verdicts were produced is the grading facet's to describe.
100//!
101//! Re-grading appends a Card version; the earlier verdicts stay with the
102//! earlier version. Grading the same runs with a different grader is a
103//! different Card.
104//!
105//! `run_results` was added in `evalhub.card/1.1`. It is an optional key, so
106//! the bump is minor: one schema document serves both identifiers, a Card
107//! declaring `evalhub.card/1.0` may carry `run_results` too, and Cards
108//! stored under 1.0 are not rewritten.
109//!
110//! ## The used set
111//!
112//! Which runs a Card used is decided per cited Eval record, and that set of
113//! `run_id`s is the Card's *used set* for the record. For each Eval record,
114//! it is the union of `attrs.runs` over the Card's `core/uses_eval`
115//! relations that resolve to that record; or, when none of those relations
116//! carries `attrs.runs`, every run of the record that is neither archived
117//! nor deleted at the moment the Card is posted. The hub fixes the used set
118//! when the Card is posted and remembers each run's content hash with it,
119//! which is how a later change to a run is detected.
120//!
121//! Against the used set:
122//!
123//! - `run_results[].eval` names the record (`{ns}/{name}`, no `@seq`) of
124//! one of the Card's `core/uses_eval` relations
125//! (`run_results_eval_unknown`).
126//! - every `run_id` in a used set has a row in that Eval, archived and
127//! deleted rows included (`run_unknown`). A run is always in the hub, so
128//! a missing row is a mistake, not an unresolved reference, and an Eval
129//! the writer may not see answers the same way as a run that does not
130//! exist.
131//! - every `run_results[].run_id` is in the used set of its `eval`
132//! (`run_not_in_used_set`).
133//!
134//! # Provenance and redaction
135//!
136//! `provenance` says where the numbers came from (`evaluation_run`,
137//! `paper`, `model_card`, …) and who ran it relative to the model
138//! (`first_party`, `third_party`). `redaction` says whether the producer
139//! removed anything before publishing, by what method, and from which fields.
140//! The hub does not redact; it only makes the producer's statement visible
141//! and searchable, and awards the `redacted` badge when `applied` is true.
142//!
143//! # Fields the hub owns
144//!
145//! `id`, `version_id`, `seq`, `label`, `content_hash`, `created_at`,
146//! `changed[]`, `badges[]` and the per-facet fingerprints are not part of
147//! this type. They are returned alongside it in API responses and are
148//! documented in `evalhub_server::api::records`.
149
150use schemars::JsonSchema;
151use serde::{Deserialize, Serialize};
152
153use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
154use crate::facet::{Env, Generation, Grading, Harness, Model, Task, Trial};
155
156/// A Card: what was measured, how, and what the score was.
157#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
158#[serde(deny_unknown_fields)]
159pub struct Card {
160 /// Schema identifier: `evalhub.card/1.1`, or `evalhub.card/1.0`, which is
161 /// still accepted. Both are described by this one schema.
162 #[schemars(extend("enum" = ["evalhub.card/1.0", "evalhub.card/1.1"]))]
163 pub schema: String,
164 /// Human-readable title.
165 pub title: String,
166 /// The software that wrote this record.
167 pub producer: Producer,
168 /// The model facet.
169 #[serde(default, skip_serializing_if = "Option::is_none")]
170 pub model: Option<Model>,
171 /// The task facet.
172 #[serde(default, skip_serializing_if = "Option::is_none")]
173 pub task: Option<Task>,
174 /// The harness facet.
175 #[serde(default, skip_serializing_if = "Option::is_none")]
176 pub harness: Option<Harness>,
177 /// The generation facet.
178 #[serde(default, skip_serializing_if = "Option::is_none")]
179 pub generation: Option<Generation>,
180 /// The trial facet.
181 #[serde(default, skip_serializing_if = "Option::is_none")]
182 pub trial: Option<Trial>,
183 /// The grading facet.
184 #[serde(default, skip_serializing_if = "Option::is_none")]
185 pub grading: Option<Grading>,
186 /// The environment facet.
187 #[serde(default, skip_serializing_if = "Option::is_none")]
188 pub env: Option<Env>,
189 /// The scores.
190 #[serde(default, skip_serializing_if = "Vec::is_empty")]
191 pub results: Vec<ResultEntry>,
192 /// Item counts the scores were computed over. Required when `results` is non-empty.
193 #[serde(default, skip_serializing_if = "Option::is_none")]
194 pub counts: Option<Counts>,
195 /// Per-run verdicts on the runs of the Evals this Card uses
196 /// (`core/uses_eval`). Each `run_id` must be in the Card's used set for
197 /// its `eval`. Added in `evalhub.card/1.1`.
198 #[serde(default, skip_serializing_if = "Vec::is_empty")]
199 pub run_results: Vec<RunResult>,
200 /// Edges to other records.
201 #[serde(default, skip_serializing_if = "Vec::is_empty")]
202 pub relations: Vec<Relation>,
203 /// Files this record refers to.
204 #[serde(default, skip_serializing_if = "Vec::is_empty")]
205 pub attachments: Vec<Attachment>,
206 /// What the producer removed before publishing.
207 #[serde(default, skip_serializing_if = "Option::is_none")]
208 pub redaction: Option<Redaction>,
209 /// Where the numbers came from and who ran the evaluation.
210 #[serde(default, skip_serializing_if = "Option::is_none")]
211 pub provenance: Option<Provenance>,
212 /// Producer-private extensions keyed by namespace.
213 #[serde(default)]
214 pub ext: Ext,
215}
216
217/// One score.
218#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
219#[serde(deny_unknown_fields)]
220pub struct ResultEntry {
221 /// Metric as a registry id, `{ns}/{name}` (for example `core/pass_rate`).
222 pub metric: String,
223 /// The score.
224 pub value: f64,
225 /// Number of samples the score was computed over.
226 #[serde(default, skip_serializing_if = "Option::is_none")]
227 pub n: Option<u64>,
228 /// How `value` was derived from the samples.
229 pub aggregation: Aggregation,
230 /// Uncertainty the producer computed.
231 #[serde(default, skip_serializing_if = "Option::is_none")]
232 pub uncertainty: Option<Uncertainty>,
233 /// The partition this score applies to (for example which grader).
234 #[serde(default, skip_serializing_if = "Option::is_none")]
235 pub by: Option<serde_json::Map<String, serde_json::Value>>,
236 /// `attachments[].path` of the per-sample rows this score summarises.
237 #[serde(default, skip_serializing_if = "Option::is_none")]
238 pub samples_ref: Option<String>,
239}
240
241/// The Card's verdict on one run for one metric.
242#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
243#[serde(deny_unknown_fields)]
244pub struct RunResult {
245 /// The Eval record the run belongs to, `{ns}/{name}` (no `@seq`: runs
246 /// belong to the record, not to a version). Must be the record of one of
247 /// this Card's `core/uses_eval` relations.
248 pub eval: String,
249 /// The run judged. Must be in this Card's used set for `eval`.
250 pub run_id: String,
251 /// Metric as a registry id, `{ns}/{name}` (for example `core/pass`).
252 pub metric: String,
253 /// The verdict as a number. At least one of `value` and `label` is present.
254 #[serde(default, skip_serializing_if = "Option::is_none")]
255 pub value: Option<f64>,
256 /// The verdict as a label (for example `pass`, `fail`). At least one of
257 /// `value` and `label` is present.
258 #[serde(default, skip_serializing_if = "Option::is_none")]
259 pub label: Option<String>,
260 /// The partition this verdict applies to (for example which grader).
261 #[serde(default, skip_serializing_if = "Option::is_none")]
262 pub by: Option<serde_json::Map<String, serde_json::Value>>,
263 /// Keys of the run's `metrics` the verdict was derived from. Informational;
264 /// the hub does not recompute.
265 #[serde(default, skip_serializing_if = "Vec::is_empty")]
266 pub from: Vec<String>,
267}
268
269/// How a score was derived from its samples.
270#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
271#[serde(rename_all = "snake_case")]
272pub enum Aggregation {
273 /// Arithmetic mean over samples.
274 Mean,
275 /// pass@k over attempts.
276 PassAtK,
277 /// Median over samples.
278 Median,
279 /// Sum over samples.
280 Sum,
281 /// A producer-defined aggregation; describe it in `ext`.
282 Custom,
283}
284
285/// Uncertainty of a score.
286#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
287#[serde(deny_unknown_fields)]
288pub struct Uncertainty {
289 /// Standard error of the score.
290 #[serde(default, skip_serializing_if = "Option::is_none")]
291 pub stderr: Option<f64>,
292 /// Confidence interval of the score.
293 #[serde(default, skip_serializing_if = "Option::is_none")]
294 pub ci: Option<ConfidenceInterval>,
295}
296
297/// A confidence interval.
298#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
299#[serde(deny_unknown_fields)]
300pub struct ConfidenceInterval {
301 /// Confidence level, for example `0.95`.
302 pub level: f64,
303 /// Lower bound.
304 pub low: f64,
305 /// Upper bound.
306 pub high: f64,
307}
308
309/// Item counts. `attempted >= completed + failed + skipped + errored`.
310#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
311#[serde(deny_unknown_fields)]
312pub struct Counts {
313 /// Items the evaluation set out to run.
314 pub attempted: u64,
315 /// Items that produced a gradable response.
316 #[serde(default)]
317 pub completed: u64,
318 /// Items that were graded as failing.
319 #[serde(default)]
320 pub failed: u64,
321 /// Items skipped by the harness.
322 #[serde(default)]
323 pub skipped: u64,
324 /// Items that errored before grading.
325 #[serde(default)]
326 pub errored: u64,
327}
328
329/// Where the numbers came from.
330#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
331#[serde(deny_unknown_fields)]
332pub struct Provenance {
333 /// What kind of source the numbers were taken from.
334 pub source_type: SourceType,
335 /// Who ran the evaluation relative to the model's maker.
336 #[serde(default, skip_serializing_if = "Option::is_none")]
337 pub evaluator_relationship: Option<EvaluatorRelationship>,
338}
339
340/// The kind of source a Card's numbers were taken from.
341#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
342#[serde(rename_all = "snake_case")]
343pub enum SourceType {
344 /// The producer ran the evaluation and recorded the result.
345 EvaluationRun,
346 /// Transcribed from a paper.
347 Paper,
348 /// Transcribed from a model card.
349 ModelCard,
350 /// Transcribed from a leaderboard.
351 Leaderboard,
352 /// Something else; describe it in `ext`.
353 Other,
354}
355
356/// Who ran the evaluation relative to the model's maker.
357#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
358#[serde(rename_all = "snake_case")]
359pub enum EvaluatorRelationship {
360 /// The model's maker evaluated its own model.
361 FirstParty,
362 /// Someone else evaluated it.
363 ThirdParty,
364}