evalhub_schema/card.rs
1//! The Card record: what was measured, how, and what the score was.
2//!
3//! A Card is the unit a harness publishes after an evaluation. It is a claim
4//! made by the producer, kept verbatim by the hub. The hub adds an id, a
5//! sequence number, a content hash, badges and fingerprints around it, but
6//! never edits the record itself; a correction is a new version.
7//!
8//! # Shape
9//!
10//! ```text
11//! Card
12//! ├── schema "evalhub.card/1.0"
13//! ├── title free text
14//! ├── producer { name, version } who wrote this record
15//! ├── model / task / harness / generation / trial / grading / env
16//! │ seven facets (see `facet`) what was measured, under what conditions
17//! ├── results[] { metric, value, n, aggregation, uncertainty, by, samples_ref }
18//! ├── counts { attempted, completed, failed, skipped, errored }
19//! ├── relations[] { type, to, attrs } edges to other records
20//! ├── attachments[] { path, sha256, size, media_type }
21//! ├── redaction { applied, method, fields }
22//! ├── provenance { source_type, evaluator_relationship }
23//! └── ext { "{ns}/{name}": { ... } } producer-private extension
24//! ```
25//!
26//! A complete example:
27//!
28//! ```json
29//! {
30//! "schema": "evalhub.card/1.0",
31//! "title": "qwen3.6-32b on single2 K4",
32//! "producer": {"name": "my-harness", "version": "0.4.1"},
33//! "model": {"id": "qwen3.6-32b", "provider": "local", "quantization": {"method": "awq", "dtype": "int4"}},
34//! "task": {"id": "single2", "version": "3", "split": "test", "n": 33, "seed": 7},
35//! "harness": {"name": "my-harness", "version": "0.4.1", "prompt_template_sha256": "…"},
36//! "generation": {"temperature": 0.0, "max_tokens": 4096},
37//! "trial": {"k": 4, "pass_at_k": true, "max_turns": 20, "sandbox": {"image": "ghcr.io/x/y", "digest": "sha256:…"}},
38//! "grading": {"graders": [{"name": "exact_match", "kind": "deterministic"}]},
39//! "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
40//! "results": [{
41//! "metric": "core/pass_rate",
42//! "value": 0.62, "n": 33,
43//! "aggregation": "mean",
44//! "uncertainty": {"stderr": 0.08, "ci": {"level": 0.95, "low": 0.45, "high": 0.77}},
45//! "by": {"grader": "exact_match"},
46//! "samples_ref": "samples.jsonl"
47//! }],
48//! "counts": {"attempted": 33, "completed": 30, "failed": 2, "skipped": 1, "errored": 0},
49//! "relations": [{"type": "core/uses_eval", "to": "alice/single2-k4@3", "attrs": {"runs": ["r1"]}}],
50//! "attachments": [{"path": "samples.jsonl", "sha256": "…", "size": 12345, "media_type": "application/x-ndjson"}],
51//! "redaction": {"applied": true, "method": "regex+llm", "fields": ["samples.jsonl:response"]},
52//! "provenance": {"source_type": "evaluation_run", "evaluator_relationship": "first_party"},
53//! "ext": {"alice/qwen-loop": {"rung": 4}}
54//! }
55//! ```
56//!
57//! # Results
58//!
59//! `results[].metric` is a registry id in the form `{ns}/{name}`
60//! (`core/pass_rate`). A metric the registry does not know is still accepted
61//! if it has that form; a bare name is rejected (`metric_id_invalid`). Whether
62//! the metric is registered is surfaced as the `metric_registered` badge, not
63//! as an error — the hub records claims, it does not gate on vocabulary.
64//!
65//! `aggregation` names how `value` was derived from the samples (`mean`,
66//! `pass_at_k`, `median`, `sum`, `custom`). `uncertainty` carries whatever the
67//! producer computed (standard error, a confidence interval). `by` records the
68//! grader or other partition the number applies to. `samples_ref` names an
69//! entry in `attachments[]` holding the per-sample rows; the hub checks that
70//! the path exists in `attachments[]` and nothing else about it.
71//!
72//! # Counts
73//!
74//! If `results` is non-empty, `counts` is required and must satisfy
75//! `attempted >= completed + failed + skipped + errored`. This is the one
76//! arithmetic rule the hub enforces, because a score without the denominator
77//! it was computed over is not comparable to anything.
78//!
79//! # Provenance and redaction
80//!
81//! `provenance` says where the numbers came from (`evaluation_run`,
82//! `paper`, `model_card`, …) and who ran it relative to the model
83//! (`first_party`, `third_party`). `redaction` says whether the producer
84//! removed anything before publishing, by what method, and from which fields.
85//! The hub does not redact; it only makes the producer's statement visible
86//! and searchable, and awards the `redacted` badge when `applied` is true.
87//!
88//! # Fields the hub owns
89//!
90//! `id`, `version_id`, `seq`, `label`, `content_hash`, `created_at`,
91//! `changed[]`, `badges[]` and the per-facet fingerprints are not part of
92//! this type. They are returned alongside it in API responses and are
93//! documented in `evalhub_server::api::records`.
94
95use schemars::JsonSchema;
96use serde::{Deserialize, Serialize};
97
98use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
99use crate::facet::{Env, Generation, Grading, Harness, Model, Task, Trial};
100
101/// A Card: what was measured, how, and what the score was.
102#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
103#[serde(deny_unknown_fields)]
104pub struct Card {
105 /// Schema identifier; must be `evalhub.card/1.0`.
106 #[schemars(extend("const" = "evalhub.card/1.0"))]
107 pub schema: String,
108 /// Human-readable title.
109 pub title: String,
110 /// The software that wrote this record.
111 pub producer: Producer,
112 /// The model facet.
113 #[serde(default, skip_serializing_if = "Option::is_none")]
114 pub model: Option<Model>,
115 /// The task facet.
116 #[serde(default, skip_serializing_if = "Option::is_none")]
117 pub task: Option<Task>,
118 /// The harness facet.
119 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub harness: Option<Harness>,
121 /// The generation facet.
122 #[serde(default, skip_serializing_if = "Option::is_none")]
123 pub generation: Option<Generation>,
124 /// The trial facet.
125 #[serde(default, skip_serializing_if = "Option::is_none")]
126 pub trial: Option<Trial>,
127 /// The grading facet.
128 #[serde(default, skip_serializing_if = "Option::is_none")]
129 pub grading: Option<Grading>,
130 /// The environment facet.
131 #[serde(default, skip_serializing_if = "Option::is_none")]
132 pub env: Option<Env>,
133 /// The scores.
134 #[serde(default, skip_serializing_if = "Vec::is_empty")]
135 pub results: Vec<ResultEntry>,
136 /// Item counts the scores were computed over. Required when `results` is non-empty.
137 #[serde(default, skip_serializing_if = "Option::is_none")]
138 pub counts: Option<Counts>,
139 /// Edges to other records.
140 #[serde(default, skip_serializing_if = "Vec::is_empty")]
141 pub relations: Vec<Relation>,
142 /// Files this record refers to.
143 #[serde(default, skip_serializing_if = "Vec::is_empty")]
144 pub attachments: Vec<Attachment>,
145 /// What the producer removed before publishing.
146 #[serde(default, skip_serializing_if = "Option::is_none")]
147 pub redaction: Option<Redaction>,
148 /// Where the numbers came from and who ran the evaluation.
149 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub provenance: Option<Provenance>,
151 /// Producer-private extensions keyed by namespace.
152 #[serde(default)]
153 pub ext: Ext,
154}
155
156/// One score.
157#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
158#[serde(deny_unknown_fields)]
159pub struct ResultEntry {
160 /// Metric as a registry id, `{ns}/{name}` (for example `core/pass_rate`).
161 pub metric: String,
162 /// The score.
163 pub value: f64,
164 /// Number of samples the score was computed over.
165 #[serde(default, skip_serializing_if = "Option::is_none")]
166 pub n: Option<u64>,
167 /// How `value` was derived from the samples.
168 pub aggregation: Aggregation,
169 /// Uncertainty the producer computed.
170 #[serde(default, skip_serializing_if = "Option::is_none")]
171 pub uncertainty: Option<Uncertainty>,
172 /// The partition this score applies to (for example which grader).
173 #[serde(default, skip_serializing_if = "Option::is_none")]
174 pub by: Option<serde_json::Map<String, serde_json::Value>>,
175 /// `attachments[].path` of the per-sample rows this score summarises.
176 #[serde(default, skip_serializing_if = "Option::is_none")]
177 pub samples_ref: Option<String>,
178}
179
180/// How a score was derived from its samples.
181#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
182#[serde(rename_all = "snake_case")]
183pub enum Aggregation {
184 /// Arithmetic mean over samples.
185 Mean,
186 /// pass@k over attempts.
187 PassAtK,
188 /// Median over samples.
189 Median,
190 /// Sum over samples.
191 Sum,
192 /// A producer-defined aggregation; describe it in `ext`.
193 Custom,
194}
195
196/// Uncertainty of a score.
197#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
198#[serde(deny_unknown_fields)]
199pub struct Uncertainty {
200 /// Standard error of the score.
201 #[serde(default, skip_serializing_if = "Option::is_none")]
202 pub stderr: Option<f64>,
203 /// Confidence interval of the score.
204 #[serde(default, skip_serializing_if = "Option::is_none")]
205 pub ci: Option<ConfidenceInterval>,
206}
207
208/// A confidence interval.
209#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
210#[serde(deny_unknown_fields)]
211pub struct ConfidenceInterval {
212 /// Confidence level, for example `0.95`.
213 pub level: f64,
214 /// Lower bound.
215 pub low: f64,
216 /// Upper bound.
217 pub high: f64,
218}
219
220/// Item counts. `attempted >= completed + failed + skipped + errored`.
221#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
222#[serde(deny_unknown_fields)]
223pub struct Counts {
224 /// Items the evaluation set out to run.
225 pub attempted: u64,
226 /// Items that produced a gradable response.
227 #[serde(default)]
228 pub completed: u64,
229 /// Items that were graded as failing.
230 #[serde(default)]
231 pub failed: u64,
232 /// Items skipped by the harness.
233 #[serde(default)]
234 pub skipped: u64,
235 /// Items that errored before grading.
236 #[serde(default)]
237 pub errored: u64,
238}
239
240/// Where the numbers came from.
241#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
242#[serde(deny_unknown_fields)]
243pub struct Provenance {
244 /// What kind of source the numbers were taken from.
245 pub source_type: SourceType,
246 /// Who ran the evaluation relative to the model's maker.
247 #[serde(default, skip_serializing_if = "Option::is_none")]
248 pub evaluator_relationship: Option<EvaluatorRelationship>,
249}
250
251/// The kind of source a Card's numbers were taken from.
252#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
253#[serde(rename_all = "snake_case")]
254pub enum SourceType {
255 /// The producer ran the evaluation and recorded the result.
256 EvaluationRun,
257 /// Transcribed from a paper.
258 Paper,
259 /// Transcribed from a model card.
260 ModelCard,
261 /// Transcribed from a leaderboard.
262 Leaderboard,
263 /// Something else; describe it in `ext`.
264 Other,
265}
266
267/// Who ran the evaluation relative to the model's maker.
268#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
269#[serde(rename_all = "snake_case")]
270pub enum EvaluatorRelationship {
271 /// The model's maker evaluated its own model.
272 FirstParty,
273 /// Someone else evaluated it.
274 ThirdParty,
275}