Skip to main content

evalhub_schema/
card.rs

1//! The Card record: what was measured, how, and what the score was.
2//!
3//! A Card is the unit a harness publishes after an evaluation. It is a claim
4//! made by the producer, kept verbatim by the hub. The hub adds an id, a
5//! sequence number, a content hash, badges and fingerprints around it, but
6//! never edits the record itself; a correction is a new version.
7//!
8//! # Shape
9//!
10//! ```text
11//! Card
12//! ├── schema        "evalhub.card/1.0"
13//! ├── title         free text
14//! ├── producer      { name, version }            who wrote this record
15//! ├── model / task / harness / generation / trial / grading / env
16//! │                 seven facets (see `facet`)   what was measured, under what conditions
17//! ├── results[]     { metric, value, n, aggregation, uncertainty, by, samples_ref }
18//! ├── counts        { attempted, completed, failed, skipped, errored }
19//! ├── relations[]   { type, to, attrs }          edges to other records
20//! ├── attachments[] { path, sha256, size, media_type }
21//! ├── redaction     { applied, method, fields }
22//! ├── provenance    { source_type, evaluator_relationship }
23//! └── ext           { "{ns}/{name}": { ... } }   producer-private extension
24//! ```
25//!
26//! A complete example:
27//!
28//! ```json
29//! {
30//!   "schema": "evalhub.card/1.0",
31//!   "title": "qwen3.6-32b on single2 K4",
32//!   "producer": {"name": "my-harness", "version": "0.4.1"},
33//!   "model": {"id": "qwen3.6-32b", "provider": "local", "quantization": {"method": "awq", "dtype": "int4"}},
34//!   "task": {"id": "single2", "version": "3", "split": "test", "n": 33, "seed": 7},
35//!   "harness": {"name": "my-harness", "version": "0.4.1", "prompt_template_sha256": "…"},
36//!   "generation": {"temperature": 0.0, "max_tokens": 4096},
37//!   "trial": {"k": 4, "pass_at_k": true, "max_turns": 20, "sandbox": {"image": "ghcr.io/x/y", "digest": "sha256:…"}},
38//!   "grading": {"graders": [{"name": "exact_match", "kind": "deterministic"}]},
39//!   "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
40//!   "results": [{
41//!     "metric": "core/pass_rate",
42//!     "value": 0.62, "n": 33,
43//!     "aggregation": "mean",
44//!     "uncertainty": {"stderr": 0.08, "ci": {"level": 0.95, "low": 0.45, "high": 0.77}},
45//!     "by": {"grader": "exact_match"},
46//!     "samples_ref": "samples.jsonl"
47//!   }],
48//!   "counts": {"attempted": 33, "completed": 30, "failed": 2, "skipped": 1, "errored": 0},
49//!   "relations": [{"type": "core/uses_eval", "to": "alice/single2-k4@3", "attrs": {"runs": ["r1"]}}],
50//!   "attachments": [{"path": "samples.jsonl", "sha256": "…", "size": 12345, "media_type": "application/x-ndjson"}],
51//!   "redaction": {"applied": true, "method": "regex+llm", "fields": ["samples.jsonl:response"]},
52//!   "provenance": {"source_type": "evaluation_run", "evaluator_relationship": "first_party"},
53//!   "ext": {"alice/qwen-loop": {"rung": 4}}
54//! }
55//! ```
56//!
57//! # Results
58//!
59//! `results[].metric` is a registry id in the form `{ns}/{name}`
60//! (`core/pass_rate`). A metric the registry does not know is still accepted
61//! if it has that form; a bare name is rejected (`metric_id_invalid`). Whether
62//! the metric is registered is surfaced as the `metric_registered` badge, not
63//! as an error — the hub records claims, it does not gate on vocabulary.
64//!
65//! `aggregation` names how `value` was derived from the samples (`mean`,
66//! `pass_at_k`, `median`, `sum`, `custom`). `uncertainty` carries whatever the
67//! producer computed (standard error, a confidence interval). `by` records the
68//! grader or other partition the number applies to. `samples_ref` names an
69//! entry in `attachments[]` holding the per-sample rows; the hub checks that
70//! the path exists in `attachments[]` and nothing else about it.
71//!
72//! # Counts
73//!
74//! If `results` is non-empty, `counts` is required and must satisfy
75//! `attempted >= completed + failed + skipped + errored`. This is the one
76//! arithmetic rule the hub enforces, because a score without the denominator
77//! it was computed over is not comparable to anything.
78//!
79//! # Provenance and redaction
80//!
81//! `provenance` says where the numbers came from (`evaluation_run`,
82//! `paper`, `model_card`, …) and who ran it relative to the model
83//! (`first_party`, `third_party`). `redaction` says whether the producer
84//! removed anything before publishing, by what method, and from which fields.
85//! The hub does not redact; it only makes the producer's statement visible
86//! and searchable, and awards the `redacted` badge when `applied` is true.
87//!
88//! # Fields the hub owns
89//!
90//! `id`, `version_id`, `seq`, `label`, `content_hash`, `created_at`,
91//! `changed[]`, `badges[]` and the per-facet fingerprints are not part of
92//! this type. They are returned alongside it in API responses and are
93//! documented in `evalhub_server::api::records`.
94
95use schemars::JsonSchema;
96use serde::{Deserialize, Serialize};
97
98use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
99use crate::facet::{Env, Generation, Grading, Harness, Model, Task, Trial};
100
101/// A Card: what was measured, how, and what the score was.
102#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
103#[serde(deny_unknown_fields)]
104pub struct Card {
105    /// Schema identifier; must be `evalhub.card/1.0`.
106    #[schemars(extend("const" = "evalhub.card/1.0"))]
107    pub schema: String,
108    /// Human-readable title.
109    pub title: String,
110    /// The software that wrote this record.
111    pub producer: Producer,
112    /// The model facet.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub model: Option<Model>,
115    /// The task facet.
116    #[serde(default, skip_serializing_if = "Option::is_none")]
117    pub task: Option<Task>,
118    /// The harness facet.
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub harness: Option<Harness>,
121    /// The generation facet.
122    #[serde(default, skip_serializing_if = "Option::is_none")]
123    pub generation: Option<Generation>,
124    /// The trial facet.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub trial: Option<Trial>,
127    /// The grading facet.
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub grading: Option<Grading>,
130    /// The environment facet.
131    #[serde(default, skip_serializing_if = "Option::is_none")]
132    pub env: Option<Env>,
133    /// The scores.
134    #[serde(default, skip_serializing_if = "Vec::is_empty")]
135    pub results: Vec<ResultEntry>,
136    /// Item counts the scores were computed over. Required when `results` is non-empty.
137    #[serde(default, skip_serializing_if = "Option::is_none")]
138    pub counts: Option<Counts>,
139    /// Edges to other records.
140    #[serde(default, skip_serializing_if = "Vec::is_empty")]
141    pub relations: Vec<Relation>,
142    /// Files this record refers to.
143    #[serde(default, skip_serializing_if = "Vec::is_empty")]
144    pub attachments: Vec<Attachment>,
145    /// What the producer removed before publishing.
146    #[serde(default, skip_serializing_if = "Option::is_none")]
147    pub redaction: Option<Redaction>,
148    /// Where the numbers came from and who ran the evaluation.
149    #[serde(default, skip_serializing_if = "Option::is_none")]
150    pub provenance: Option<Provenance>,
151    /// Producer-private extensions keyed by namespace.
152    #[serde(default)]
153    pub ext: Ext,
154}
155
156/// One score.
157#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
158#[serde(deny_unknown_fields)]
159pub struct ResultEntry {
160    /// Metric as a registry id, `{ns}/{name}` (for example `core/pass_rate`).
161    pub metric: String,
162    /// The score.
163    pub value: f64,
164    /// Number of samples the score was computed over.
165    #[serde(default, skip_serializing_if = "Option::is_none")]
166    pub n: Option<u64>,
167    /// How `value` was derived from the samples.
168    pub aggregation: Aggregation,
169    /// Uncertainty the producer computed.
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub uncertainty: Option<Uncertainty>,
172    /// The partition this score applies to (for example which grader).
173    #[serde(default, skip_serializing_if = "Option::is_none")]
174    pub by: Option<serde_json::Map<String, serde_json::Value>>,
175    /// `attachments[].path` of the per-sample rows this score summarises.
176    #[serde(default, skip_serializing_if = "Option::is_none")]
177    pub samples_ref: Option<String>,
178}
179
180/// How a score was derived from its samples.
181#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
182#[serde(rename_all = "snake_case")]
183pub enum Aggregation {
184    /// Arithmetic mean over samples.
185    Mean,
186    /// pass@k over attempts.
187    PassAtK,
188    /// Median over samples.
189    Median,
190    /// Sum over samples.
191    Sum,
192    /// A producer-defined aggregation; describe it in `ext`.
193    Custom,
194}
195
196/// Uncertainty of a score.
197#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
198#[serde(deny_unknown_fields)]
199pub struct Uncertainty {
200    /// Standard error of the score.
201    #[serde(default, skip_serializing_if = "Option::is_none")]
202    pub stderr: Option<f64>,
203    /// Confidence interval of the score.
204    #[serde(default, skip_serializing_if = "Option::is_none")]
205    pub ci: Option<ConfidenceInterval>,
206}
207
208/// A confidence interval.
209#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
210#[serde(deny_unknown_fields)]
211pub struct ConfidenceInterval {
212    /// Confidence level, for example `0.95`.
213    pub level: f64,
214    /// Lower bound.
215    pub low: f64,
216    /// Upper bound.
217    pub high: f64,
218}
219
220/// Item counts. `attempted >= completed + failed + skipped + errored`.
221#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
222#[serde(deny_unknown_fields)]
223pub struct Counts {
224    /// Items the evaluation set out to run.
225    pub attempted: u64,
226    /// Items that produced a gradable response.
227    #[serde(default)]
228    pub completed: u64,
229    /// Items that were graded as failing.
230    #[serde(default)]
231    pub failed: u64,
232    /// Items skipped by the harness.
233    #[serde(default)]
234    pub skipped: u64,
235    /// Items that errored before grading.
236    #[serde(default)]
237    pub errored: u64,
238}
239
240/// Where the numbers came from.
241#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
242#[serde(deny_unknown_fields)]
243pub struct Provenance {
244    /// What kind of source the numbers were taken from.
245    pub source_type: SourceType,
246    /// Who ran the evaluation relative to the model's maker.
247    #[serde(default, skip_serializing_if = "Option::is_none")]
248    pub evaluator_relationship: Option<EvaluatorRelationship>,
249}
250
251/// The kind of source a Card's numbers were taken from.
252#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
253#[serde(rename_all = "snake_case")]
254pub enum SourceType {
255    /// The producer ran the evaluation and recorded the result.
256    EvaluationRun,
257    /// Transcribed from a paper.
258    Paper,
259    /// Transcribed from a model card.
260    ModelCard,
261    /// Transcribed from a leaderboard.
262    Leaderboard,
263    /// Something else; describe it in `ext`.
264    Other,
265}
266
267/// Who ran the evaluation relative to the model's maker.
268#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
269#[serde(rename_all = "snake_case")]
270pub enum EvaluatorRelationship {
271    /// The model's maker evaluated its own model.
272    FirstParty,
273    /// Someone else evaluated it.
274    ThirdParty,
275}